From 58b90a61c1af9f7e7f5a511252cf2dd2b8c6e924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:58:00 +0000 Subject: [PATCH 001/108] Declare policyengine-uk's whole-population formulas and share the engine-block machinery (#931) Per-clone engine blocks in the dense measure step, and the per-clone spine batches that follow, run policyengine-uk over part of the pool, so the formulas that read beyond their own households see the part. This adds the pieces the next commits wire in. - microcosm.frame.adapters.engine_population is country-agnostic: numpy and duck-typed simulations, no engine import. It holds the pool-share correction c_b = f_b * S_b / S with its zero rules, checks (a) and (b), and injection into a simulation, its baseline and every sibling, refused once a branch exists or a recorder is armed. A per-instance _run_formula recorder, which core's clone rebinds into later branches, and a known-periods walk feed a judge over declarations, parameter predicates and pass policies. A held value is judged unless the simulation holding it ran it after arming, and a run reaches the position window by the last year its own period covers. The verdict carries names, periods and booleans only. Household batches, the nesting check, the scatter back to one engine's order, a float32 share bound that follows numpy's pairwise-sum leaves and counts the adds forming each key, and a per-row bound for values that add share terms to others complete it. - microcosm.frame.adapters.policyengine_uk_population classifies pe-uk 2.174.1's 19 WHOLE_POPULATION_VARIABLES: three corrected shares, two stored inputs, eight zero at policy, attendance inert through private_school_vat_change, six refused ranks. It adds the window from REBALANCING_YEARS (2026 on), the constructor's structural reform factories, sha256 pins of the sources the classes rest on, the spine batch column, and uk_engine_population(), which imports pe-uk and validates only when called, failing closed. Nothing imports either module yet, so no kernel, stage or target hash moves and no output changes. Engine-free tests drive the machinery on fake core-style simulations, covering each guard condition a small change to the module could drop. Engine tests hold the declaration to the installed engine: list, adds, pins and what they cover, predicates at 2024 and 2025 and moved off current law, reform factories, window, the readers of attendance and private-school VAT, and drift lists of axis-free reductions and of whole-array branches in formulas and the engine functions they reach. They correct two synthetic clones that differ to one engine over the pool within the float32 bounds, household_tax per row, with household-local outputs bitwise equal. A build test holds the batch column and the share inputs to the build's own constants. Co-Authored-By: Claude Opus 5.5 --- .../test_uk_engine_population_declaration.py | 30 + .../frame/adapters/engine_population.py | 1963 +++++++++++++++++ .../adapters/policyengine_uk_population.py | 272 +++ .../uk/test_policyengine_uk_population.py | 1083 +++++++++ .../shared/test_engine_population.py | 1744 +++++++++++++++ 5 files changed, 5092 insertions(+) create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_engine_population_declaration.py create mode 100644 packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py create mode 100644 packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk_population.py create mode 100644 packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_population.py create mode 100644 packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_engine_population_declaration.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_engine_population_declaration.py new file mode 100644 index 0000000000..6bc9f8791f --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_engine_population_declaration.py @@ -0,0 +1,30 @@ +"""The build's constants name what the UK engine-population declaration does. + +microcosm-frame cannot import microcosm-build, so the two things both name +are held together here (microcosm#931): the household column spine engine +calls batch by, which must be the ladder frame's clone index, and the frame +columns each share formula's allocation key adds, which the measure step +sums for the pool-share correction. +""" + +from __future__ import annotations + +from microcosm.build.uk_runtime.full_measure import UK_WEIGHT_SHARE_FORMULA_INPUTS +from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column +from microcosm.frame.adapters.policyengine_uk_population import ( + UK_ENGINE_POPULATION, + UK_SPINE_ENGINE_BATCH_COLUMN, +) + + +def test_spine_engine_calls_batch_by_the_ladder_households_clone_index(): + assert UK_SPINE_ENGINE_BATCH_COLUMN == ladder_clone_index_column("household") + + +def test_the_share_inputs_the_build_sums_are_the_declared_key_adds(): + declared = { + formula: UK_ENGINE_POPULATION.key_adds[key] + for formula, key in UK_ENGINE_POPULATION.corrected.items() + } + # Tuples: the same columns in the same order. + assert dict(UK_WEIGHT_SHARE_FORMULA_INPUTS) == declared diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py new file mode 100644 index 0000000000..6bb32603b7 --- /dev/null +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py @@ -0,0 +1,1963 @@ +"""Engine blocks that stand for one pool (microcosm#931). + +A rules engine run over part of a population, one engine block per clone in a +dense measure step or one batch per clone in a spine call, works out the +formulas that read beyond their own households over that part rather than +over the pool. This module holds the country-agnostic pieces that make such a +block stand for the pool; each country passes a declaration of which of its +formulas read beyond their households (:class:`PopulationDeclaration`). + +Pool shares. A formula ``v = x / sum(x * w)``, optionally times a constant, +evaluated over a block whose engine weights are ``f_b * w`` returns +``x / (f_b * S_b)``, where ``S_b`` is the block's ``sum(x * w)`` at unscaled +weights; over the pool it returns ``x / S``. So the pool's value is the +block's own value times ``c_b = f_b * S_b / S``, one scalar for every row of +the block. An engine that carries each key and the weights to a later year by +one factor per row keeps ``c_b`` at every year, which +:func:`check_uniform_growth` checks. :func:`inject_pool_shares` sets the +block's own value times ``c_b`` as an input, at each period the block +computes, in the block's simulation and its baseline, before any branch is +made and before the guard is armed; :meth:`InjectedShares.apply` does the +same in every other simulation the block builds. :func:`pool_share_rtol` +bounds the result's float32 gap from one engine over the pool, and +:func:`pool_share_atol` the gap of a value that adds share terms to others. + +The zero rules. A block whose keys are all zero has ``S_b = 0``; its rows +take zero, as they do in the pool, whatever ``S``. A block with +``S_b = 0`` but nonzero keys, or a pool with ``S = 0`` and a block with +``S_b != 0``, sends the formula down a fallback branch no scalar can +correct, so it is refused (:class:`PoolShareError`). + +The guard. A :class:`FormulaRunRecorder` binds itself to each simulation's +``_run_formula`` (a per-instance method, never a class-level patch). A +policyengine-core clone rebinds instance-bound methods to the copy, so +branches made after arming record too, and the recorder sees values the +engine later drops and branches it pops. A walk of the holders' known periods +cross-checks it: a value a simulation holds is judged unless that simulation +ran it after arming. :func:`judge` holds both against the declaration under +a :class:`PassPolicy`: the corrected formulas must be inputs at every +period, the stored ones too, the zero-at-policy ones may run only under +their parameter predicate and only to all-zero values, an inert one may run +only through a sink that comes out all zero, the refused ones never run, +nothing runs in the declaration's position window (by the last year a +run's own period covers), and no structural reform the engine's +constructor applies is switched on. The verdict carries names, periods and +booleans only. + +Batches. :func:`household_batches` splits a frame by a household column +(whole households, groups nested in households) and +:func:`scatter_by_entity_ids` puts each batch's results back in the order one +engine over the frame returns them. + +Simulations are duck-typed policyengine-core simulations (``calculate``, +``set_input``, ``get_holder``, ``baseline``, ``branches``, ``branch_name``, +``tax_benefit_system.parameters`` and the instance-bindable +``_run_formula``). No engine is imported when this module loads; a +declaration is checked against its engine only when a caller asks +(:func:`validate_declaration`). +""" + +from __future__ import annotations + +import functools +import hashlib +import importlib +import importlib.util +import inspect +import json +import math +import operator +import types +import weakref +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from importlib import metadata +from pathlib import Path +from types import MappingProxyType +from typing import Any + +import numpy as np +import pandas as pd + +__all__ = [ + "FLOAT32_UNIT_ROUNDOFF", + "MEASURE_PASS", + "SPINE_BATCHED_PASS", + "AllOf", + "AnyOf", + "FormulaRun", + "FormulaRunRecorder", + "HouseholdBatch", + "InertRule", + "InjectedShares", + "ParameterTest", + "PassPolicy", + "PoolShare", + "PoolShareError", + "PopulationDeclaration", + "PopulationDeclarationError", + "PopulationGuardError", + "PopulationVerdict", + "PositionWindow", + "Violation", + "all_of", + "any_of", + "assert_groups_nest_in_households", + "check_key_sums", + "check_uniform_growth", + "derived_watched_periods", + "factor_spread", + "float32_sum_roundings", + "household_batches", + "inject_pool_shares", + "judge", + "pool_share_atol", + "pool_share_rtol", + "pool_shares", + "scatter_by_entity_ids", + "source_sha256", + "validate_declaration", +] + + +class PoolShareError(ValueError): + """A block's pool-share correction cannot be applied exactly.""" + + +class PopulationDeclarationError(ValueError): + """An engine no longer matches its declaration of whole-population formulas.""" + + +class PopulationGuardError(RuntimeError): + """A block or batch engine computed what only the pool defines.""" + + def __init__(self, verdict: PopulationVerdict) -> None: + self.verdict = verdict + super().__init__(verdict.describe()) + + +# --------------------------------------------------------------------------- +# The float32 bound +# --------------------------------------------------------------------------- + +#: The unit roundoff of float32, the precision policyengine-core stores and +#: computes variables in. +FLOAT32_UNIT_ROUNDOFF = 2.0**-24 + +#: Roundings in one share row besides the two sums and the adds that form a +#: key from several inputs: the row products ``x * w`` in the block's and the +#: pool's sums (2), the divisions by the sums (2), the products by a constant +#: such as an aggregate land value (2), the float32 storage of the key's +#: inputs (at most one rounding of a nonnegative key) and of ``w`` against +#: the float64 frame sums ``c_b`` is formed from, for ``S_b`` and ``S`` (4), +#: the product by ``c_b`` and its float32 cast (1), and float64 slack (2). +_SHARE_ROUNDINGS = 13 + + +@functools.cache +def float32_sum_roundings(n: int) -> int: + """The most roundings numpy's float32 sum of ``n`` terms chains into one term. + + numpy's ``pairwise_sum`` adds fewer than eight terms in order. Up to 128 + terms it keeps eight accumulators, each starting from one term and adding + every eighth term after it, combines them in three levels and then adds + the at most seven terms left over in order: up to 15 + 3 + 7 roundings. + Above 128 it splits the array at half its length, rounded down to a + multiple of eight, and adds the two halves' sums: one rounding per level. + So the leaf alone contributes about 25 roundings whatever ``n`` is, and a + bound of ``log2(n)`` roundings (which a plain recursive halving would + give) is not valid. + """ + + n = int(n) + if n < 8: + return max(n - 1, 0) + if n <= 128: + return (n // 8 - 1) + 3 + (n % 8) + half = n // 2 + half -= half % 8 + return 1 + max(float32_sum_roundings(half), float32_sum_roundings(n - half)) + + +def pool_share_rtol( + block_rows: int, + pool_rows: int, + *, + key_inputs: int, + projection_years: int = 0, +) -> float: + """A valid bound on ``|v_block * c_b / v_pool - 1|`` for one row. + + For a share formula whose nonnegative key adds ``key_inputs`` inputs (the + length of the key's ``adds``), evaluated in float32 over a block of + ``block_rows`` households and over a pool of ``pool_rows``. It counts the + two sums' roundings (:func:`float32_sum_roundings`) and the row's own + roundings. It counts the engine's float32 adds that form each row's key + from its inputs: ``key_inputs - 1`` roundings in every term of the + block's sum and of the pool's, which the float64 frame sums ``c_b`` is + formed from do not carry and which need not cancel between the two (so + ``2 * (key_inputs - 1)``). And at a period the engine projects + ``projection_years`` years from the dataset, by one factor per row + (:func:`check_uniform_growth`), it counts one rounding per year in each + row's key and weight, in the block's and the pool's sums. A first-order + bound in the unit roundoff. A value that adds share-dependent terms to + others (a household's taxes, say) is bounded per row by + :func:`pool_share_atol`, not by this relative to itself. + """ + + if int(key_inputs) < 1: + raise ValueError("A key adds at least one input.") + roundings = ( + float32_sum_roundings(block_rows) + + float32_sum_roundings(pool_rows) + + _SHARE_ROUNDINGS + + 2 * (int(key_inputs) - 1) + + 4 * max(int(projection_years), 0) + ) + return roundings * FLOAT32_UNIT_ROUNDOFF + + +def pool_share_atol( + share_terms: Any, + magnitude: Any, + *, + share_rtol: float, + additions: int, +) -> np.ndarray: + """A valid per-row bound on ``|v_block - v_pool|`` for a value that adds + share-dependent terms to household-local ones. + + ``share_terms`` is each row's sum of the absolute share-dependent terms + the value adds (a household's business rates and corporate SDLT, say) + and ``share_rtol`` their bound from :func:`pool_share_rtol`. + ``magnitude`` bounds the absolute value of every partial sum the value's + additions form for the row; the sum of its terms' absolute values does. + ``additions`` counts the additions, the final float32 storage included, + at which the block's and the pool's partial sums can round apart. The + household-local terms are equal in both engines, so the gap is the share + terms' own, ``share_rtol * share_terms``, plus a rounding of up to + ``u * magnitude`` in each engine at each such addition. Relative to the + value itself the gap can be large, where the value nets large terms to a + small one. + """ + + if int(additions) < 0: + raise ValueError("additions cannot be negative.") + share = np.abs(np.asarray(share_terms, dtype=np.float64)) + size = np.abs(np.asarray(magnitude, dtype=np.float64)) + return float(share_rtol) * share + 2 * int(additions) * FLOAT32_UNIT_ROUNDOFF * size + + +# --------------------------------------------------------------------------- +# Periods and values +# --------------------------------------------------------------------------- + + +def _period_key(period: Any) -> str: + return str(period) + + +#: The years an eternity period covers: all of them. +_EVERY_YEAR = (-math.inf, math.inf) + + +def _period_span(period: Any) -> tuple[float, float] | None: + """The first and last calendar years ``period`` covers, or None. + + A policyengine-core period gives its own start and stop; its eternity + unit covers every year. A year as an integer covers itself. A string is + read only in the forms ``YYYY``, ``YYYY-MM`` and ``YYYY-MM-DD`` (one + year) and ``ETERNITY``; any other form gives None, which callers treat + as covering every year. + """ + + if isinstance(period, bool): + return None + if isinstance(period, (int, np.integer)): + return (int(period), int(period)) + if getattr(period, "unit", None) == "eternity": + return _EVERY_YEAR + try: + first, last = period.start.year, period.stop.year + except Exception: + first = last = None + if isinstance(first, (int, float)) and isinstance(last, (int, float)): + return (first, last) + text = str(period).strip() + if text.lower() == "eternity": + return _EVERY_YEAR + parts = text.split("-") + if ( + 1 <= len(parts) <= 3 + and len(parts[0]) == 4 + and all(part.isdigit() for part in parts) + and all(len(part) == 2 for part in parts[1:]) + ): + return (int(parts[0]), int(parts[0])) + return None + + +def _values(result: Any) -> np.ndarray: + # A pandas or microdf series gives its array, in its own dtype, without + # microdf's warning that ``.values`` drops the weights. + return np.asarray(result) + + +def _all_zero(result: Any) -> bool | None: + try: + return bool(np.all(_values(result) == 0)) + except Exception: + # A result that cannot be read as an array is not known to be zero. + return None + + +# --------------------------------------------------------------------------- +# Pool shares +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PoolShare: + """One block's correction of one share formula to the pool's denominator. + + ``block_key_sum`` is the block's ``sum(x * w)`` and ``pool_key_sum`` the + pool's, both at unscaled weights; ``weight_scale`` is the factor ``f_b`` + the block's engine weights carry (1 when they are unscaled). + """ + + formula: str + key_variable: str + block_key_sum: float + pool_key_sum: float + weight_scale: float = 1.0 + + def __post_init__(self) -> None: + for name in ("block_key_sum", "pool_key_sum", "weight_scale"): + value = float(getattr(self, name)) + if not math.isfinite(value): + raise PoolShareError(f"{self.formula}: {name} must be finite.") + object.__setattr__(self, name, value) + if not self.weight_scale > 0.0: + raise PoolShareError(f"{self.formula}: weight_scale must be positive.") + + @property + def keys_must_be_zero(self) -> bool: + """Whether the block's rows take zero: its key sum is zero.""" + + return self.block_key_sum == 0.0 + + def factor(self) -> float: + """``c_b = f_b * S_b / S``, under the zero rules. + + A zero block sum gives 0: the block's rows take zero, which is the + pool's value only if every key in the block is zero, as the injection + and :func:`check_key_sums` require. A zero pool sum with a nonzero + block sum is refused. + """ + + if self.block_key_sum == 0.0: + return 0.0 + if self.pool_key_sum == 0.0: + raise PoolShareError( + f"{self.formula}: the pool's {self.key_variable} total is zero " + "while this block's is not, so the pool's engine takes a " + "fallback branch no factor reproduces." + ) + return self.weight_scale * self.block_key_sum / self.pool_key_sum + + def as_receipt(self) -> dict[str, Any]: + """The correction, as a ratio of block-weighted totals.""" + + return { + "formula": self.formula, + "key_variable": self.key_variable, + "correction": self.factor(), + } + + +def pool_shares( + block_key_sums: Mapping[Any, Mapping[str, float]], + weight_scales: Mapping[Any, float] | None = None, + *, + key_variables: Mapping[str, str], +) -> dict[Any, dict[str, PoolShare]]: + """Every block's :class:`PoolShare` for every corrected formula. + + ``block_key_sums[block][formula]`` is the block's ``sum(x * w)`` at + unscaled weights; the pool's is their sum, since the blocks partition the + pool. ``weight_scales[block]`` is the block's engine weight factor + ``f_b``; without it every block's engine sees unscaled weights. + ``key_variables`` maps each formula to the key variable it divides by. + """ + + formulas = sorted(key_variables) + blocks = list(block_key_sums) + if not blocks: + raise PoolShareError("pool shares need at least one block.") + for block in blocks: + if sorted(block_key_sums[block]) != formulas: + raise PoolShareError( + f"block {block!r} carries key sums for " + f"{sorted(block_key_sums[block])}, not {formulas}." + ) + if weight_scales is not None and set(weight_scales) != set(blocks): + raise PoolShareError("every block needs exactly one engine weight factor.") + pool = { + formula: math.fsum(float(block_key_sums[block][formula]) for block in blocks) + for formula in formulas + } + return { + block: { + formula: PoolShare( + formula=formula, + key_variable=key_variables[formula], + block_key_sum=float(block_key_sums[block][formula]), + pool_key_sum=pool[formula], + weight_scale=( + 1.0 if weight_scales is None else float(weight_scales[block]) + ), + ) + for formula in formulas + } + for block in blocks + } + + +def check_key_sums( + simulation: Any, + shares: Mapping[str, PoolShare], + period: Any, + *, + weight_variable: str, + rtol: float = 1e-5, +) -> dict[str, float]: + """Check (a): the engine's own ``sum(x * w)`` is ``f_b * S_b``. + + Read at ``period`` (the dataset year) from the engine's own key and + weight arrays, so a key column the engine did not load, a stored total + or a change to the key's inputs fails here. The tolerance is relative to + ``sum(|x * w|)``. A share whose block sum is zero requires every key in + the block to be zero. Returns each formula's relative gap, a ratio of + block-weighted totals. + """ + + weights = _values(simulation.calculate(weight_variable, period)).astype(np.float64) + gaps: dict[str, float] = {} + keys: dict[str, np.ndarray] = {} + for formula, share in sorted(shares.items()): + if share.key_variable not in keys: + keys[share.key_variable] = _values( + simulation.calculate(share.key_variable, period) + ).astype(np.float64) + key = keys[share.key_variable] + if key.shape != weights.shape: + raise PoolShareError( + f"{share.key_variable} and {weight_variable} do not align at {period}." + ) + if share.keys_must_be_zero: + if np.any(key != 0): + raise PoolShareError( + f"{formula}: the block's {share.key_variable} total is zero " + "but not every key is, so the block's engine takes a " + "fallback branch no factor corrects." + ) + gaps[formula] = 0.0 + continue + products = key * weights + scale = float(np.sum(np.abs(products))) + expected = share.weight_scale * share.block_key_sum + gap = abs(float(np.sum(products)) - expected) / max(scale, abs(expected)) + if not gap <= rtol: + raise PoolShareError( + f"{formula}: the engine's {share.key_variable} total at {period} " + "is not the block's frame total, so the correction would not " + "reach the pool's denominator." + ) + gaps[formula] = gap + return gaps + + +def check_uniform_growth( + simulation: Any, + names: Iterable[str], + start: Any, + end: Any, + *, + rtol: float = 1e-5, +) -> dict[str, float | None]: + """Check (b): the engine carries each of ``names`` by one factor per row. + + From ``start`` (the dataset year) to ``end``: every row that is zero at + ``start`` stays zero, and every other row grows by the same factor. That + keeps ``S_b / S`` at ``end`` what it is at ``start``, so a ``c_b`` from + dataset-year totals holds there. Pass each key itself, not only its + parts, and the weight variable. Returns each name's factor, the ratio of + its absolute totals (None where no row carries it). + """ + + factors: dict[str, float | None] = {} + for name in names: + then = _values(simulation.calculate(name, start)).astype(np.float64) + now = _values(simulation.calculate(name, end)).astype(np.float64) + if then.shape != now.shape: + raise PoolShareError(f"{name} changes shape between {start} and {end}.") + carried = then != 0 + if np.any(now[~carried] != 0): + raise PoolShareError( + f"{name} is zero at {start} on rows where it is not at {end}, so " + "the engine does not carry it by one factor per row." + ) + if not carried.any(): + factors[name] = None + continue + factor = float(np.sum(np.abs(now[carried])) / np.sum(np.abs(then[carried]))) + expected = factor * then[carried] + if np.any(np.abs(now[carried] - expected) > rtol * np.abs(expected)): + raise PoolShareError( + f"The engine does not carry {name} from {start} to {end} by one " + "factor per row, so a dataset-year correction does not hold there." + ) + factors[name] = factor + return factors + + +def factor_spread( + factors_by_block: Mapping[Any, Mapping[str, float | None]], +) -> dict[str, float]: + """Each name's spread of factors across blocks: ``(max - min) / |mean|``. + + Blocks whose factor is None (no row carries the name) are left out. The + factors are the engine's parameter-driven growth, equal in every block + up to float32 rounding. + """ + + names = sorted({name for factors in factors_by_block.values() for name in factors}) + spread: dict[str, float] = {} + for name in names: + values = [ + float(factors[name]) + for factors in factors_by_block.values() + if factors.get(name) is not None + ] + if len(values) < 2: + spread[name] = 0.0 + continue + centre = abs(math.fsum(values) / len(values)) + spread[name] = (max(values) - min(values)) / centre if centre else math.inf + return spread + + +class InjectedShares: + """The pool's share arrays one block's simulations take, at each period.""" + + def __init__( + self, + *, + shares: Mapping[str, PoolShare], + periods: Sequence[Any], + own: Mapping[tuple[str, str], np.ndarray], + corrected: Mapping[tuple[str, str], np.ndarray], + references: Mapping[tuple[str, str], np.ndarray], + ) -> None: + self._shares = dict(shares) + self._periods = tuple(periods) + self._own = dict(own) + self._corrected = dict(corrected) + self._references = dict(references) + + @property + def periods(self) -> tuple[str, ...]: + return tuple(_period_key(period) for period in self._periods) + + @property + def factors(self) -> dict[str, float]: + return {formula: share.factor() for formula, share in self._shares.items()} + + def apply(self, simulation: Any) -> None: + """Set the same arrays into ``simulation`` and its baseline. + + For another simulation the block builds (an annual-income or + counterfactual sibling): its own share values must equal, bitwise, + those the arrays were made from, so the same correction is the + pool's value there too. As with :func:`inject_pool_shares`, apply + before arming a recorder on the sibling and before it has branches. + """ + + for target in _with_baseline(simulation): + _refuse_unready(target) + _refuse_injected(target, self._shares, self._periods) + own = _own_values(target, self._shares, self._periods) + for key, values in own.items(): + if not np.array_equal(values, self._own[key]): + formula, period = key + raise PoolShareError( + f"{formula} at {period} differs in a simulation the " + "block builds, so the block's correction is not its " + "pool value there." + ) + for target in _with_baseline(simulation): + _set_corrected(target, self._corrected, self._periods) + + def matches(self, simulation: Any) -> bool: + """Whether ``simulation`` and its baseline still hold the block's keys + and weights at every period, bitwise (checked when the block ends).""" + + for target in _with_baseline(simulation): + for (name, period), reference in self._references.items(): + period_value = self._period_value(period) + values = _values(target.calculate(name, period_value)) + if not np.array_equal(values, reference): + return False + return True + + def as_receipt(self) -> dict[str, Any]: + return { + "periods": list(self.periods), + "corrections": { + formula: factor for formula, factor in sorted(self.factors.items()) + }, + } + + def _period_value(self, key: str) -> Any: + for period in self._periods: + if _period_key(period) == key: + return period + raise KeyError(key) + + +def _with_baseline(simulation: Any) -> list[Any]: + targets = [simulation] + baseline = getattr(simulation, "baseline", None) + if baseline is not None and baseline is not simulation: + targets.append(baseline) + return targets + + +def _armed_by_a_recorder(simulation: Any) -> bool: + bound = getattr(simulation, "__dict__", {}).get("_run_formula") + function = getattr(bound, "__func__", None) + owner = getattr(function, "_microcosm_recorder", None) + recorder = owner() if owner is not None else None + return recorder is not None and recorder._recording + + +def _refuse_unready(simulation: Any) -> None: + """Refuse to set shares into a simulation with live branches, which keep + their own values and would go on computing the block's own shares, or + one a recorder is armed on, which would record the reads of its own + values as runs.""" + + branches = getattr(simulation, "branches", None) + if isinstance(branches, Mapping) and branches: + raise PoolShareError( + "The simulation has live branches, which would keep computing " + "the block's own shares; set the pool's shares before any branch " + "is made." + ) + if _armed_by_a_recorder(simulation): + raise PoolShareError( + "A recorder is armed on the simulation; set the pool's shares " + "before arming it." + ) + + +def _input_keys(simulation: Any, name: str) -> set[str]: + holder = simulation.get_holder(name) + getter = getattr(holder, "get_input_periods", None) + if getter is None: + return set() + branch = getattr(simulation, "branch_name", "default") + return {_period_key(period) for period in getter(branch)} + + +def _refuse_injected( + simulation: Any, shares: Mapping[str, PoolShare], periods: Sequence[Any] +) -> None: + for formula in shares: + inputs = _input_keys(simulation, formula) + for period in periods: + if _period_key(period) in inputs: + raise PoolShareError( + f"{formula} is already an input at {period}; its own value " + "cannot be read, and correcting it twice would compound." + ) + + +def _own_values( + simulation: Any, shares: Mapping[str, PoolShare], periods: Sequence[Any] +) -> dict[tuple[str, str], np.ndarray]: + own: dict[tuple[str, str], np.ndarray] = {} + for formula in sorted(shares): + for period in periods: + values = np.array(_values(simulation.calculate(formula, period))) + if not np.all(np.isfinite(values)): + raise PoolShareError(f"{formula} at {period} is not finite.") + own[(formula, _period_key(period))] = values + return own + + +def _set_corrected( + simulation: Any, + corrected: Mapping[tuple[str, str], np.ndarray], + periods: Sequence[Any], +) -> None: + by_key = {_period_key(period): period for period in periods} + for (formula, key), values in corrected.items(): + simulation.set_input(formula, by_key[key], values.copy()) + + +def inject_pool_shares( + simulation: Any, + shares: Mapping[str, PoolShare], + periods: Iterable[Any], + *, + weight_variable: str, +) -> InjectedShares: + """Set each share formula to its own value times ``c_b`` at every period. + + Into ``simulation`` and its baseline, before anything reads the shares, + before any branch is made and before a recorder is armed (each is + refused). The own values are the engine's (read first, for every formula + and period, before any is set), so no row alignment is involved; a block + whose key sum is zero takes zeros, and every key in it must be zero. + The returned :class:`InjectedShares` sets the same arrays into the other + simulations the block builds. + """ + + periods = tuple({_period_key(period): period for period in periods}.values()) + if not periods: + raise PoolShareError("pool shares need at least one period.") + targets = _with_baseline(simulation) + for target in targets: + _refuse_unready(target) + _refuse_injected(target, shares, periods) + own = _own_values(simulation, shares, periods) + for target in targets[1:]: + for key, values in _own_values(target, shares, periods).items(): + if not np.array_equal(values, own[key]): + raise PoolShareError( + f"{key[0]} at {key[1]} differs between the simulation and " + "its baseline, so one correction cannot serve both." + ) + references: dict[tuple[str, str], np.ndarray] = {} + for period in periods: + references[(weight_variable, _period_key(period))] = np.array( + _values(simulation.calculate(weight_variable, period)) + ) + for share in shares.values(): + key = (share.key_variable, _period_key(period)) + if key not in references: + references[key] = np.array( + _values(simulation.calculate(share.key_variable, period)) + ) + if share.keys_must_be_zero and np.any(references[key] != 0): + raise PoolShareError( + f"{share.formula}: the block's {share.key_variable} total is " + f"zero but not every key is at {period}, so the block's " + "engine takes a fallback branch no factor corrects." + ) + corrected: dict[tuple[str, str], np.ndarray] = {} + for (formula, key), values in own.items(): + share = shares[formula] + if share.keys_must_be_zero: + corrected[(formula, key)] = np.zeros_like(values) + else: + corrected[(formula, key)] = ( + values.astype(np.float64) * share.factor() + ).astype(values.dtype) + for target in targets: + _set_corrected(target, corrected, periods) + return InjectedShares( + shares=shares, + periods=periods, + own=own, + corrected=corrected, + references=references, + ) + + +# --------------------------------------------------------------------------- +# Parameter predicates +# --------------------------------------------------------------------------- + +_OPERATORS = { + "eq": operator.eq, + "ne": operator.ne, + "lt": operator.lt, + "le": operator.le, + "gt": operator.gt, + "ge": operator.ge, +} + + +def _parameter_value(node: Any, path: str) -> Any: + for part in path.split("."): + try: + node = getattr(node, part) + except Exception: + try: + node = node[part] + except Exception as error: + raise AttributeError(f"No parameter {path}.") from error + return node + + +@dataclass(frozen=True) +class ParameterTest: + """A test of one parameter at a period: ``path op value``. + + ``op`` is one of ``eq``, ``ne``, ``lt``, ``le``, ``gt``, ``ge``, or + ``eq_path``, which compares the parameter with the one at the path given + as ``value`` (a reform's parameter with its baseline's, say). + """ + + path: str + op: str + value: bool | int | float | str | None = None + + def __post_init__(self) -> None: + if self.op not in (*_OPERATORS, "eq_path"): + raise ValueError(f"Unknown parameter test operator {self.op!r}.") + if self.op == "eq_path" and not isinstance(self.value, str): + raise ValueError("An eq_path test compares with a parameter path.") + + def holds(self, parameters: Any) -> bool: + left = _parameter_value(parameters, self.path) + if self.op == "eq_path": + return bool(left == _parameter_value(parameters, str(self.value))) + return bool(_OPERATORS[self.op](left, self.value)) + + def paths(self) -> tuple[str, ...]: + return (self.path, str(self.value)) if self.op == "eq_path" else (self.path,) + + def to_json(self) -> dict[str, Any]: + return {"parameter": self.path, "op": self.op, "value": self.value} + + +@dataclass(frozen=True) +class AllOf: + """Every test holds.""" + + tests: tuple[Any, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "tests", tuple(self.tests)) + if not self.tests: + raise ValueError("A predicate needs at least one test.") + + def holds(self, parameters: Any) -> bool: + return all(test.holds(parameters) for test in self.tests) + + def paths(self) -> tuple[str, ...]: + return tuple(path for test in self.tests for path in test.paths()) + + def to_json(self) -> dict[str, Any]: + return {"all_of": [test.to_json() for test in self.tests]} + + +@dataclass(frozen=True) +class AnyOf: + """At least one test holds.""" + + tests: tuple[Any, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "tests", tuple(self.tests)) + if not self.tests: + raise ValueError("A predicate needs at least one test.") + + def holds(self, parameters: Any) -> bool: + return any(test.holds(parameters) for test in self.tests) + + def paths(self) -> tuple[str, ...]: + return tuple(path for test in self.tests for path in test.paths()) + + def to_json(self) -> dict[str, Any]: + return {"any_of": [test.to_json() for test in self.tests]} + + +#: A predicate over a simulation's parameters at a period. +Predicate = ParameterTest | AllOf | AnyOf + + +def all_of(*tests: Predicate) -> AllOf: + return AllOf(tests) + + +def any_of(*tests: Predicate) -> AnyOf: + return AnyOf(tests) + + +def _predicate_holds_in(simulation: Any, predicate: Predicate, period: Any) -> bool: + try: + parameters = simulation.tax_benefit_system.parameters(period) + return bool(predicate.holds(parameters)) + except Exception: + # A predicate whose parameters cannot be read does not hold. + return False + + +# --------------------------------------------------------------------------- +# The declaration +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class InertRule: + """When a formula that reads beyond its households cannot reach a value. + + The formula's value reaches the population's totals only through + ``sink``, a difference that is zero everywhere while ``predicate`` holds + (a policy equal to its baseline). The formula may run in a block only if + the sink ran wherever it did and came out all zero, the predicate held, + none of ``carriers`` (which read a level the formula feeds) ran, and + neither the formula, its ``levels`` nor a carrier was requested. + """ + + sink: str + levels: tuple[str, ...] + carriers: tuple[str, ...] + predicate: Predicate + + def __post_init__(self) -> None: + object.__setattr__(self, "levels", tuple(self.levels)) + object.__setattr__(self, "carriers", tuple(self.carriers)) + + def unrequestable(self, name: str) -> frozenset[str]: + return frozenset({name, *self.levels, *self.carriers}) + + def to_json(self) -> dict[str, Any]: + return { + "sink": self.sink, + "levels": sorted(self.levels), + "carriers": sorted(self.carriers), + "predicate": self.predicate.to_json(), + } + + +@dataclass(frozen=True) +class PositionWindow: + """Years in which an engine's values depend on a record's position. + + Any block or batch work in a year from ``first_year`` on is refused: + a requested or measured period there, or any formula run there after + the guard is armed. ``source`` names the engine's own statement of the + years (``module:attribute``, a ``range`` starting at ``first_year``). + """ + + first_year: int + source: str + reason: str + + def contains(self, period: Any) -> bool: + """Whether ``period`` reaches the window: its last year is + ``first_year`` or later. An eternity period covers every year, so it + does; so does a period whose years cannot be read.""" + + return self.reaches(_period_span(period)) + + def reaches(self, span: tuple[float, float] | None) -> bool: + """Whether years ``(first, last)`` reach the window; None, years not + known, does.""" + + return span is None or span[1] >= self.first_year + + def to_json(self) -> dict[str, Any]: + return {"first_year": self.first_year, "source": self.source} + + +def _frozen_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]: + return MappingProxyType(dict(sorted(dict(value).items()))) + + +def _names(value: Iterable[str]) -> tuple[str, ...]: + names = tuple(value) + if len(set(names)) != len(names): + raise ValueError(f"Names repeat in {names}.") + return names + + +@dataclass(frozen=True, eq=False) +class PopulationDeclaration: + """An engine's formulas that read beyond their own households, by class. + + - ``corrected``: share formulas (to the key variable each divides by), + set to the pool's value in every block (:func:`inject_pool_shares`); + ``key_adds`` names each key's inputs, which the frame sums. + - ``stored``: formulas the data supplies as inputs at every period. + - ``zero_at_policy``: formulas that are zero while their predicate holds. + - ``inert``: stored formulas that may instead run under an + :class:`InertRule`, in a pass that allows it. + - ``refused``: formulas no block may run (ranks and medians). + + ``list_source`` names the engine's own list (``module:attribute``), which + the classes must cover exactly; ``observed`` names formulas recorded for + receipts but not judged; ``constructor_factories`` names the structural + reform factories the engine's constructor applies, which must each return + None; ``digests`` pins the sources the classification rests on + (:func:`source_sha256`). ``reviewed_version`` is the engine release the + classification was reviewed against; validation rests on the digests. + """ + + engine: str + distribution: str + reviewed_version: str + list_source: str + weight_variable: str + corrected: Mapping[str, str] + key_adds: Mapping[str, tuple[str, ...]] + stored: tuple[str, ...] + zero_at_policy: Mapping[str, Predicate] + inert: Mapping[str, InertRule] + refused: tuple[str, ...] + window: PositionWindow + observed: tuple[str, ...] = () + constructor_factories: tuple[str, ...] = () + digests: Mapping[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "corrected", _frozen_mapping(self.corrected)) + object.__setattr__( + self, + "key_adds", + _frozen_mapping( + {key: tuple(adds) for key, adds in dict(self.key_adds).items()} + ), + ) + object.__setattr__(self, "zero_at_policy", _frozen_mapping(self.zero_at_policy)) + object.__setattr__(self, "inert", _frozen_mapping(self.inert)) + object.__setattr__(self, "digests", _frozen_mapping(self.digests)) + for name in ("stored", "refused", "observed", "constructor_factories"): + object.__setattr__(self, name, _names(getattr(self, name))) + classes = { + "corrected": set(self.corrected), + "stored": set(self.stored), + "zero_at_policy": set(self.zero_at_policy), + "refused": set(self.refused), + } + for first, first_names in classes.items(): + for second, second_names in classes.items(): + if first < second and first_names & second_names: + raise ValueError( + f"{sorted(first_names & second_names)} are both " + f"{first} and {second}." + ) + if not set(self.inert) <= set(self.stored): + raise ValueError("An inert formula must also be declared stored.") + missing_keys = set(self.corrected.values()) - set(self.key_adds) + if missing_keys: + raise ValueError(f"Keys {sorted(missing_keys)} name no inputs.") + + def classified(self) -> frozenset[str]: + return frozenset( + { + *self.corrected, + *self.stored, + *self.zero_at_policy, + *self.inert, + *self.refused, + } + ) + + def watched(self) -> frozenset[str]: + """Every formula the guard records: the classified ones, the inert + rules' sinks, levels and carriers, and the observed ones.""" + + names = set(self.classified()) | set(self.observed) + for rule in self.inert.values(): + names |= {rule.sink, *rule.levels, *rule.carriers} + return frozenset(names) + + def run_predicates(self) -> dict[str, Predicate]: + """The predicate checked at every run of each formula that has one.""" + + predicates: dict[str, Predicate] = dict(self.zero_at_policy) + for name, rule in self.inert.items(): + predicates[name] = rule.predicate + predicates[rule.sink] = rule.predicate + return predicates + + def to_json(self) -> dict[str, Any]: + return { + "engine": self.engine, + "distribution": self.distribution, + "reviewed_version": self.reviewed_version, + "list_source": self.list_source, + "weight_variable": self.weight_variable, + "corrected": dict(self.corrected), + "key_adds": {key: list(adds) for key, adds in self.key_adds.items()}, + "stored": sorted(self.stored), + "zero_at_policy": { + name: predicate.to_json() + for name, predicate in self.zero_at_policy.items() + }, + "inert": {name: rule.to_json() for name, rule in self.inert.items()}, + "refused": sorted(self.refused), + "observed": sorted(self.observed), + "window": self.window.to_json(), + "constructor_factories": sorted(self.constructor_factories), + "digests": dict(self.digests), + } + + @property + def sha256(self) -> str: + text = json.dumps(self.to_json(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _resolve_reference(reference: str) -> Any: + module_name, _, qualified = reference.partition(":") + if not qualified: + raise ValueError(f"{reference!r} is not of the form 'module:attribute'.") + value: Any = importlib.import_module(module_name) + for part in qualified.split("."): + value = getattr(value, part) + return value + + +def source_sha256(reference: str) -> str: + """The sha256 of the source a declaration pins. + + ``package/relative/path.py`` names a file inside an installed package (its + bytes are hashed); ``module:qualified.name`` names a function or class + (the text ``inspect.getsource`` returns is hashed). + """ + + if ":" in reference: + source = inspect.getsource(_resolve_reference(reference)) + return hashlib.sha256(source.encode("utf-8")).hexdigest() + package, _, relative = reference.partition("/") + spec = importlib.util.find_spec(package) + if spec is None or not spec.submodule_search_locations: + raise FileNotFoundError(f"No installed package {package!r}.") + for location in spec.submodule_search_locations: + path = Path(location) / relative + if path.is_file(): + return hashlib.sha256(path.read_bytes()).hexdigest() + raise FileNotFoundError(f"{reference} is not in the installed package.") + + +def _is_formula(variable: Any) -> bool: + return bool(getattr(variable, "formulas", None)) + + +def _list_of(value: Any) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + return (value,) + return tuple(value) + + +def validate_declaration(declaration: PopulationDeclaration, system: Any) -> None: + """Fail unless ``system`` (the engine's tax-benefit system) still matches. + + Its own list equals the declared classes; every named variable exists; + every share formula is a formula over its key, every key a sum of plain + inputs of the key's entity exactly as declared; the pinned sources are + unchanged; the window's source starts at the window; every constructor + factory resolves; every predicate's parameters exist. Callers validate + when they first need the declaration, never at import. + """ + + problems: list[str] = [] + try: + listed = frozenset(_resolve_reference(declaration.list_source)) + except Exception as error: + problems.append(f"{declaration.list_source} cannot be read ({error}).") + else: + classified = declaration.classified() + if listed != classified: + problems.append( + f"{declaration.list_source} lists {sorted(listed - classified)} " + "the declaration does not classify, and the declaration " + f"classifies {sorted(classified - listed)} it does not list." + ) + variables = system.variables + needed = ( + set(declaration.watched()) + | set(declaration.corrected.values()) + | {add for adds in declaration.key_adds.values() for add in adds} + | {declaration.weight_variable} + ) + absent = sorted(name for name in needed if name not in variables) + if absent: + problems.append(f"The engine has no variables {absent}.") + weight = variables.get(declaration.weight_variable) + for formula, key in declaration.corrected.items(): + formula_variable = variables.get(formula) + key_variable = variables.get(key) + if formula_variable is None or key_variable is None: + continue + if not _is_formula(formula_variable): + problems.append(f"{formula} is no longer a formula.") + entity = key_variable.entity.key + if formula_variable.entity.key != entity or ( + weight is not None and weight.entity.key != entity + ): + problems.append( + f"{formula}, {key} and {declaration.weight_variable} no longer " + "share an entity." + ) + if _is_formula(key_variable) or _list_of( + getattr(key_variable, "subtracts", None) + ): + problems.append(f"{key} is no longer a plain sum of its inputs.") + adds = _list_of(getattr(key_variable, "adds", None)) + if adds != declaration.key_adds[key]: + problems.append( + f"{key} adds {list(adds)}, not the declared " + f"{list(declaration.key_adds[key])}." + ) + for add in adds: + leaf = variables.get(add) + if leaf is None: + continue + if ( + _is_formula(leaf) + or _list_of(getattr(leaf, "adds", None)) + or _list_of(getattr(leaf, "subtracts", None)) + or leaf.entity.key != entity + ): + problems.append(f"{add} is no longer a plain {entity} input.") + for reference, expected in declaration.digests.items(): + try: + actual = source_sha256(reference) + except Exception as error: + problems.append(f"{reference} cannot be read ({error}).") + continue + if actual != expected: + problems.append(f"{reference} has changed since it was reviewed.") + window = declaration.window + try: + years = _resolve_reference(window.source) + except Exception as error: + problems.append(f"{window.source} cannot be read ({error}).") + else: + if ( + not isinstance(years, range) + or years.step != 1 + or years.start != window.first_year + ): + problems.append( + f"{window.source} no longer starts the window at {window.first_year}." + ) + for reference in declaration.constructor_factories: + try: + factory = _resolve_reference(reference) + except Exception as error: + problems.append(f"{reference} cannot be read ({error}).") + continue + if not callable(factory): + problems.append(f"{reference} is not callable.") + for name, predicate in sorted(declaration.run_predicates().items()): + for path in predicate.paths(): + try: + _parameter_value(system.parameters, path) + except AttributeError: + problems.append(f"{name}: the engine has no parameter {path}.") + if problems: + raise PopulationDeclarationError( + f"{declaration.engine}'s whole-population declaration no longer " + "matches the installed engine:\n - " + "\n - ".join(problems) + ) + + +# --------------------------------------------------------------------------- +# Passes and verdicts +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PassPolicy: + """How a pass treats the guard. + + ``inert_allowed``: whether an inert rule may stand in for a stored + input. ``on_violation``: ``record`` returns a verdict that is not clean + (the caller marks its result inexact); ``raise`` raises + :class:`PopulationGuardError`. + """ + + name: str + inert_allowed: bool + on_violation: str + + def __post_init__(self) -> None: + if self.on_violation not in ("record", "raise"): + raise ValueError("on_violation is 'record' or 'raise'.") + + +#: The dense measure step: attendance and the other stored formulas must be +#: inputs, and a violation is recorded, leaving the blocks inexact (María's +#: ruling of 2026-11-06, microcosm#931). +MEASURE_PASS = PassPolicy("measure", inert_allowed=False, on_violation="record") +#: A spine engine call batched by clone: an inert formula may run under its +#: rule, and a violation raises (the same ruling). +SPINE_BATCHED_PASS = PassPolicy( + "spine_batched", inert_allowed=True, on_violation="raise" +) + + +@dataclass(frozen=True) +class Violation: + """One thing a block or batch did that only the pool defines.""" + + kind: str + name: str | None = None + period: str | None = None + + def as_receipt(self) -> dict[str, Any]: + return {"kind": self.kind, "name": self.name, "period": self.period} + + def describe(self) -> str: + where = "" if self.period is None else f"@{self.period}" + return f"{self.kind} {self.name or ''}{where}".strip() + + +#: The most violations a verdict lists; the rest are counted out as truncated. +_VIOLATION_LIMIT = 50 + + +@dataclass(frozen=True) +class PopulationVerdict: + """What one block or batch's simulations did, judged. Names, periods and + booleans only: no value.""" + + pass_name: str + violations: tuple[Violation, ...] + truncated: bool + classes: Mapping[str, tuple[str, ...]] + window_first_year: int + declaration_sha256: str + engine: str + engine_version: str | None + + @property + def clean(self) -> bool: + return not self.violations + + def as_receipt(self) -> dict[str, Any]: + window = [v for v in self.violations if v.kind.startswith("window_")] + refused = sorted( + { + v.name + for v in self.violations + if v.kind.startswith("refused_") and v.name + } + ) + return { + "pass": self.pass_name, + "clean": self.clean, + "declaration_sha256": self.declaration_sha256, + "engine": {"package": self.engine, "version": self.engine_version}, + "classes": {name: list(names) for name, names in self.classes.items()}, + "refused": refused, + "window": { + "first_year": self.window_first_year, + "violations": [violation.as_receipt() for violation in window], + }, + "violations": [violation.as_receipt() for violation in self.violations], + "truncated": self.truncated, + } + + def describe(self) -> str: + if self.clean: + return f"pass {self.pass_name!r}: clean" + listed = "; ".join(violation.describe() for violation in self.violations) + more = " (and more)" if self.truncated else "" + return ( + f"pass {self.pass_name!r}: a block or batch engine computed what " + f"only the pool defines: {listed}{more}." + ) + + +# --------------------------------------------------------------------------- +# The recorder +# --------------------------------------------------------------------------- + +_TOKEN_ATTRIBUTE = "_microcosm_formula_run_token" + + +@dataclass(frozen=True) +class FormulaRun: + """One run of a watched formula: in which simulation (the recorder's + token for it), at which period, whether it produced a value, whether + that value was all zero, and whether its predicate held, read at the run + from that simulation's own parameters.""" + + simulation: int + name: str + period: str + produced: bool + all_zero: bool | None + predicate_holds: bool | None + + +class FormulaRunRecorder: + """Records the formulas simulations run once it is armed on them. + + :meth:`arm` binds a recording ``_run_formula`` to a simulation, its + baseline and its live branches, as instance methods. A policyengine-core + clone rebinds instance-bound methods to the copy, so any branch made + afterwards (``get_branch``, household subsets) records too. Watched + formulas are recorded in full; every other run only by period, for the + position window. Each simulation is told apart by a token checked + against a weak reference, so a copy that inherits its source's token, or + a simulation that reuses a collected one's id, gets its own. + """ + + def __init__( + self, + watched: Iterable[str], + *, + predicates: Mapping[str, Predicate] | None = None, + ) -> None: + self.watched = frozenset(watched) + self._predicates = dict(predicates or {}) + unknown = set(self._predicates) - self.watched + if unknown: + raise ValueError(f"Predicates for unwatched formulas {sorted(unknown)}.") + self._runs: list[FormulaRun] = [] + self._periods: dict[str, set[str]] = {} + self._spans: dict[str, tuple[float, float] | None] = {} + self._simulations: dict[int, weakref.ref] = {} + self._next_token = 0 + self._recording = False + recorder = self + + def run_formula(simulation, variable, population, period): + result = type(simulation)._run_formula( + simulation, variable, population, period + ) + if recorder._recording: + recorder._record(simulation, variable, period, result) + return result + + run_formula._microcosm_recorder = weakref.ref(self) + self._run_formula = run_formula + + @classmethod + def for_declaration(cls, declaration: PopulationDeclaration) -> FormulaRunRecorder: + return cls(declaration.watched(), predicates=declaration.run_predicates()) + + @property + def runs(self) -> tuple[FormulaRun, ...]: + return tuple(self._runs) + + def periods_run(self) -> dict[str, frozenset[str]]: + """Every formula that ran after arming, by period.""" + + return {period: frozenset(names) for period, names in self._periods.items()} + + def period_span(self, period: str) -> tuple[float, float] | None: + """The first and last years a recorded period covers, read from the + period the engine ran at (None where they could not be read).""" + + return self._spans.get(period) + + def arm(self, simulation: Any) -> None: + """Record from now on in ``simulation``, its baseline and its branches. + + Every one is checked before any is bound, so a refusal leaves them, + and the recorder, as they were. + """ + + targets = list(_walk([simulation])) + for current in targets: + bound = vars(current).get("_run_formula") + function = getattr(bound, "__func__", None) + if bound is not None and function is not self._run_formula: + owner = getattr(function, "_microcosm_recorder", None) + previous = owner() if owner is not None else None + if owner is None or (previous is not None and previous._recording): + raise ValueError( + "The simulation already carries its own _run_formula, " + "which arming would bypass." + ) + for current in targets: + bound = vars(current).get("_run_formula") + function = getattr(bound, "__func__", None) + if function is not self._run_formula or bound.__self__ is not current: + current._run_formula = types.MethodType(self._run_formula, current) + self._token(current) + self._recording = True + + def is_armed(self, simulation: Any) -> bool: + bound = vars(simulation).get("_run_formula") + return ( + self._recording + and getattr(bound, "__func__", None) is self._run_formula + and bound.__self__ is simulation + ) + + def disarm(self) -> None: + """Stop recording and unbind from every simulation recorded so far. + + A copy made after arming that never ran a formula keeps the binding, + which then only passes through. + """ + + self._recording = False + for reference in self._simulations.values(): + simulation = reference() + if simulation is None: + continue + state = vars(simulation) + bound = state.get("_run_formula") + if getattr(bound, "__func__", None) is self._run_formula: + del state["_run_formula"] + state.pop(_TOKEN_ATTRIBUTE, None) + + def token(self, simulation: Any) -> int | None: + """The token runs in ``simulation`` carry, if it ran or was armed.""" + + token = vars(simulation).get(_TOKEN_ATTRIBUTE) + reference = self._simulations.get(token) if token is not None else None + if reference is not None and reference() is simulation: + return token + return None + + def _token(self, simulation: Any) -> int: + token = self.token(simulation) + if token is not None: + return token + token = self._next_token + self._next_token += 1 + self._simulations[token] = weakref.ref(simulation) + vars(simulation)[_TOKEN_ATTRIBUTE] = token + return token + + def _record(self, simulation: Any, variable: Any, period: Any, result: Any) -> None: + name = variable.name + key = _period_key(period) + token = self._token(simulation) + self._periods.setdefault(key, set()).add(name) + if key not in self._spans: + self._spans[key] = _period_span(period) + if name not in self.watched: + return + produced = result is not None + predicate = self._predicates.get(name) + self._runs.append( + FormulaRun( + simulation=token, + name=name, + period=key, + produced=produced, + all_zero=_all_zero(result) if produced else None, + predicate_holds=( + None + if predicate is None + else _predicate_holds_in(simulation, predicate, period) + ), + ) + ) + + +# --------------------------------------------------------------------------- +# The known-periods walk and the judge +# --------------------------------------------------------------------------- + + +def _walk(simulations: Iterable[Any]) -> Iterator[Any]: + """Each simulation once, with its baseline and live branches, depth first.""" + + seen: set[int] = set() + held: list[Any] = [] + stack = [simulation for simulation in simulations][::-1] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + held.append(current) + yield current + children: list[Any] = [] + branches = getattr(current, "branches", None) + if isinstance(branches, Mapping): + children.extend(branches.values()) + baseline = getattr(current, "baseline", None) + if baseline is not None: + children.append(baseline) + stack.extend(reversed(children)) + + +def _derived( + simulations: Iterable[Any], names: Iterable[str] +) -> Iterator[tuple[Any, str, Any]]: + """(simulation, name, period) for every value held that is not an input.""" + + names = tuple(names) + for simulation in _walk(simulations): + for name in names: + holder = simulation.get_holder(name) + known = holder.get_known_periods() + if not known: + continue + inputs = _input_keys(simulation, name) + for period in known: + if _period_key(period) not in inputs: + yield simulation, name, period + + +def derived_watched_periods( + simulations: Iterable[Any], names: Iterable[str] +) -> dict[str, tuple[str, ...]]: + """The periods at which each of ``names`` holds a calculated value, in any + of ``simulations``, their baselines or their live branches.""" + + found: dict[str, set[str]] = {} + for _simulation, name, period in _derived(simulations, names): + found.setdefault(name, set()).add(_period_key(period)) + return {name: tuple(sorted(periods)) for name, periods in sorted(found.items())} + + +def _held_all_zero(simulation: Any, name: str, period: Any) -> bool: + holder = simulation.get_holder(name) + try: + values = holder.get_array(period, getattr(simulation, "branch_name", "default")) + except TypeError: + values = holder.get_array(period) + return values is not None and bool(_all_zero(values)) + + +def _engine_version(distribution: str) -> str | None: + try: + return metadata.version(distribution) + except metadata.PackageNotFoundError: + return None + + +def judge( + declaration: PopulationDeclaration, + policy: PassPolicy, + recorder: FormulaRunRecorder, + simulations: Sequence[Any], + periods: Iterable[Any], + requested: Iterable[tuple[str, Any]] = (), +) -> PopulationVerdict: + """Judge what a block or batch's simulations did against the declaration. + + ``simulations`` are the ones the block or batch built (its baselines and + live branches are walked with them); every one must be armed with + ``recorder``. ``periods`` are the periods the block computes at, where + the corrected and stored formulas must be inputs. ``requested`` lists the + ``(variable, period)`` pairs the caller asked for. Under a ``raise`` + policy a verdict that is not clean raises :class:`PopulationGuardError`. + """ + + tops = list(_walk_tops(simulations)) + targets = [target for top in tops for target in _with_baseline(top)] + periods = tuple({_period_key(period): period for period in periods}.values()) + requested = tuple(requested) + window = declaration.window + violations: list[Violation] = [] + seen: set[tuple[str, str | None, str | None]] = set() + + def flag(kind: str, name: str | None = None, period: Any = None) -> None: + key = (kind, name, None if period is None else _period_key(period)) + if key not in seen: + seen.add(key) + violations.append(Violation(*key)) + + for target in targets: + if not recorder.is_armed(target): + flag("not_armed") + for period in periods: + if window.contains(period): + flag("window_period", None, period) + for name, period in requested: + if window.contains(period): + flag("window_request", name, period) + for period, names in sorted(recorder.periods_run().items()): + # The years the engine's own period covered, recorded at the run. + if window.reaches(recorder.period_span(period)): + for name in sorted(names): + flag("window_run", name, period) + for top in tops: + parameters = top.tax_benefit_system.parameters + for reference in declaration.constructor_factories: + factory = _resolve_reference(reference) + for period in periods: + if factory(parameters, period) is not None: + flag("constructor_reform", reference, period) + + runs: dict[str, list[FormulaRun]] = {} + recorded: set[tuple[int, str, str]] = set() + for run in recorder.runs: + if run.produced: + runs.setdefault(run.name, []).append(run) + recorded.add((run.simulation, run.name, run.period)) + derived: dict[str, list[tuple[Any, Any]]] = {} + judged = sorted(declaration.watched()) + for simulation, name, period in _derived(tops, judged): + derived.setdefault(name, []).append((simulation, period)) + + def judged_at_run(simulation: Any, name: str, period: Any) -> bool: + # A held value was judged when it ran only if this simulation ran it + # at this period after arming; a run elsewhere says nothing of it. + token = recorder.token(simulation) + return token is not None and (token, name, _period_key(period)) in recorded + + def ran_at(name: str) -> set[str]: + return {run.period for run in runs.get(name, ())} | { + _period_key(period) for _simulation, period in derived.get(name, ()) + } + + def missing_inputs(name: str) -> list[Any]: + return [ + period + for period in periods + if any( + _period_key(period) not in _input_keys(target, name) + for target in targets + ) + ] + + classes: dict[str, set[str]] = { + "corrected": set(), + "stored": set(), + "zero_at_policy": set(), + "inert": set(), + } + for name in sorted(declaration.corrected): + missing = missing_inputs(name) + for period in missing: + flag("corrected_not_input", name, period) + for period in sorted(ran_at(name)): + flag("corrected_derived", name, period) + if not missing and not ran_at(name): + classes["corrected"].add(name) + for name in sorted(declaration.refused): + for period in sorted(ran_at(name)): + flag("refused_ran", name, period) + for requested_name, period in requested: + if requested_name == name: + flag("refused_request", name, period) + for name, predicate in sorted(declaration.zero_at_policy.items()): + good = True + for run in runs.get(name, ()): + if not (run.predicate_holds and run.all_zero): + good = False + flag("zero_at_policy", name, run.period) + for simulation, period in derived.get(name, ()): + if judged_at_run(simulation, name, period): + continue + if not ( + _predicate_holds_in(simulation, predicate, period) + and _held_all_zero(simulation, name, period) + ): + good = False + flag("zero_at_policy", name, period) + if good and ran_at(name): + classes["zero_at_policy"].add(name) + for name in sorted(declaration.stored): + missing = missing_inputs(name) + ran = sorted(ran_at(name)) + if not missing and not ran: + classes["stored"].add(name) + continue + rule = declaration.inert.get(name) if policy.inert_allowed else None + if rule is None: + for period in missing: + flag("stored_not_input", name, period) + for period in ran: + flag("stored_derived", name, period) + continue + before = len(violations) + _judge_inert(name, rule, ran, runs, derived, requested, flag, judged_at_run) + if len(violations) == before and ran: + classes["inert"].add(name) + + truncated = len(violations) > _VIOLATION_LIMIT + verdict = PopulationVerdict( + pass_name=policy.name, + violations=tuple(violations[:_VIOLATION_LIMIT]), + truncated=truncated, + classes={name: tuple(sorted(names)) for name, names in classes.items()}, + window_first_year=window.first_year, + declaration_sha256=declaration.sha256, + engine=declaration.distribution, + engine_version=_engine_version(declaration.distribution), + ) + if not verdict.clean and policy.on_violation == "raise": + raise PopulationGuardError(verdict) + return verdict + + +def _walk_tops(simulations: Iterable[Any]) -> Iterator[Any]: + seen: set[int] = set() + for simulation in simulations: + if simulation is not None and id(simulation) not in seen: + seen.add(id(simulation)) + yield simulation + + +def _judge_inert( + name: str, + rule: InertRule, + ran: Sequence[str], + runs: Mapping[str, Sequence[FormulaRun]], + derived: Mapping[str, Sequence[tuple[Any, Any]]], + requested: Sequence[tuple[str, Any]], + flag: Any, + judged_at_run: Any, +) -> None: + """Flag every way an inert formula's runs could reach a value.""" + + for run in runs.get(name, ()): + if not run.predicate_holds: + flag("inert_predicate", name, run.period) + sink_runs = runs.get(rule.sink, ()) + # The periods the sink ran or is held at, in any simulation; each held + # value is judged unless the simulation holding it ran it after arming. + sink_periods = {run.period for run in sink_runs} + for simulation, period in derived.get(rule.sink, ()): + sink_periods.add(_period_key(period)) + if judged_at_run(simulation, rule.sink, period): + continue + if not ( + _predicate_holds_in(simulation, rule.predicate, period) + and _held_all_zero(simulation, rule.sink, period) + ): + flag("inert_sink", rule.sink, period) + for run in sink_runs: + if run.period in ran and not (run.predicate_holds and run.all_zero): + flag("inert_sink", rule.sink, run.period) + for period in ran: + if period not in sink_periods: + flag("inert_without_sink", name, period) + for carrier in rule.carriers: + periods = {run.period for run in runs.get(carrier, ())} | { + _period_key(period) for _simulation, period in derived.get(carrier, ()) + } + for period in sorted(periods): + flag("inert_carrier", carrier, period) + unrequestable = rule.unrequestable(name) + for requested_name, period in requested: + if requested_name in unrequestable: + flag("inert_request", requested_name, period) + + +# --------------------------------------------------------------------------- +# Batches +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, eq=False) +class HouseholdBatch: + """The households carrying one value of the batch column, and their people.""" + + key: Any + households: np.ndarray + persons: np.ndarray + + +def assert_groups_nest_in_households( + bundle: Any, *, household: str = "household" +) -> None: + """Refuse a frame with a group that has members in two households. + + Batches take whole households; such a group would be split between + batches and each batch would simulate it with part of its members. + """ + + schema = bundle.schema + person = bundle.table(schema.person_entity) + households = person[schema.membership_column(household)].to_numpy() + for group in schema.group_entities: + if group == household: + continue + pairs = pd.DataFrame( + { + "group": person[schema.membership_column(group)].to_numpy(), + "household": households, + } + ).drop_duplicates() + if pairs["group"].duplicated().any(): + raise ValueError( + f"{group} units must nest in households: one with members in " + "two households would be split between household batches." + ) + + +def household_batches( + bundle: Any, column: str, *, household: str = "household" +) -> tuple[HouseholdBatch, ...]: + """Whole-household batches of ``bundle``, one per value of ``column``. + + In ascending order of the value, each with boolean masks over the + bundle's household and person rows (people keep the bundle's order). + Groups must nest in households. + """ + + assert_groups_nest_in_households(bundle, household=household) + schema = bundle.schema + table = bundle.table(household) + if column not in table.columns: + raise ValueError(f"The {household} table carries no {column!r} column.") + values = table[column].to_numpy() + if pd.isna(values).any(): + raise ValueError(f"{column!r} is missing for some {household} rows.") + ids = table[schema.id_column(household)].to_numpy() + membership = ( + bundle.table(schema.person_entity)[schema.membership_column(household)] + ).to_numpy() + batches = [] + for key in np.unique(values): + in_batch = values == key + batches.append( + HouseholdBatch( + key=key.item() if hasattr(key, "item") else key, + households=in_batch, + persons=np.isin(membership, ids[in_batch]), + ) + ) + return tuple(batches) + + +def scatter_by_entity_ids( + bundle: Any, + entity: str, + parts: Iterable[tuple[np.ndarray, np.ndarray]], +) -> np.ndarray: + """One array over ``bundle``'s ``entity`` rows from per-batch arrays. + + ``parts`` holds each batch's ``(ids, values)``, the ids being the batch + frame's own. The result follows the bundle's rows: people in the + bundle's order and groups in ascending id order (a frame keeps its group + tables sorted), which is the order one engine over the bundle returns. + Every row must be filled exactly once. + """ + + target = np.asarray(bundle.table(entity)[bundle.schema.entity_id_column(entity)]) + parts = [(np.asarray(ids), np.asarray(values)) for ids, values in parts] + if not parts: + raise ValueError("Nothing to scatter.") + for ids, values in parts: + if ids.shape != values.shape: + raise ValueError("A batch's ids and values do not align.") + dtypes = {values.dtype for _ids, values in parts} + if len(dtypes) != 1: + raise ValueError(f"The batches return {entity} values of different types.") + ids = np.concatenate([ids for ids, _values in parts]) + values = np.concatenate([values for _ids, values in parts]) + if ids.shape != target.shape: + raise ValueError( + f"The batches hold a different number of {entity} rows from the bundle." + ) + order = np.argsort(target, kind="stable") + ordered = target[order] + if np.any(ordered[1:] == ordered[:-1]): + raise ValueError(f"The bundle's {entity} ids repeat.") + positions = np.searchsorted(ordered, ids) + found = positions < len(ordered) + found[found] = ordered[positions[found]] == ids[found] + if not found.all(): + raise ValueError(f"A batch returns {entity} ids the bundle does not hold.") + slots = order[positions] + if len(np.unique(slots)) != len(slots): + raise ValueError(f"Two batches return the same {entity} row.") + result = np.empty(len(target), dtype=values.dtype) + result[slots] = values + return result diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk_population.py b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk_population.py new file mode 100644 index 0000000000..86591b3efe --- /dev/null +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk_population.py @@ -0,0 +1,272 @@ +"""policyengine-uk's whole-population formulas, declared for engine blocks. + +policyengine-uk lists the formulas that read beyond their own households in +``utils/household_subset.py`` (``WHOLE_POPULATION_VARIABLES``): a weighted +total, a weighted rank or median, or records spread by weight. Over an engine +block or batch that holds part of the pool they read the part, not the pool +(microcosm#931). :data:`UK_ENGINE_POPULATION` classifies each one for the +shared machinery in :mod:`microcosm.frame.adapters.engine_population`, +reviewed against policyengine-uk 2.174.1: + +- corrected: ``shareholding``, ``corporate_land_value`` and + ``consumption_shareholding`` allocate by ``x / sum(x * w)``, so a block's + own value times its pool-mass factor and key sums is the pool's; +- stored: ``months_since_last_birthday`` and ``attends_private_school`` + are inputs the spine stores; +- zero at policy: the labour supply response, the employer NI incidence + responses, the carbon tax and the four contributed budget changes are zero + while their parameters are (checked at each run); +- inert: before the spine stores attendance, a batched call may compute it + only where it reaches household totals through + ``private_school_vat_change``, which is zero while the private school VAT + rate equals its baseline's; +- refused: the household income and wealth deciles and the relative poverty + flags rank or take a median over the simulated population, which no factor + corrects. + +The window: policyengine-uk's constructor draws which Universal Credit +claimants are new claimants by benefit-unit position for the rebalancing +years (``scenarios/uc_reform.py``, ``REBALANCING_YEARS``), so blocks and +batches refuse any work from 2026 on. + +Kept apart from the adapter module, as the concept mapping is, so that +editing the declaration never moves the adapter's source hash. Nothing here +imports policyengine-uk when the module loads; :func:`uk_engine_population` +imports it, validates the declaration against the installed engine and +fails closed. +""" + +from __future__ import annotations + +from typing import Any + +from microcosm.frame.adapters.engine_population import ( + InertRule, + ParameterTest, + PopulationDeclaration, + PositionWindow, + all_of, + any_of, + validate_declaration, +) + +__all__ = [ + "UK_ENGINE_POPULATION", + "UK_SPINE_ENGINE_BATCH_COLUMN", + "uk_engine_population", +] + +#: The household column spine engine calls batch by: the clone index the +#: ladder frame carries (``ladder_clone_index_column("household")``), so a +#: batch is one clone's households. +UK_SPINE_ENGINE_BATCH_COLUMN = "household_clone_index" + +_COICOP_CONSUMPTION = ( + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", +) + +_EMPLOYER_NI = "gov.contrib.policyengine.employer_ni" +_BUDGET = "gov.contrib.policyengine.budget" + + +def _zero(path: str) -> ParameterTest: + return ParameterTest(path, "eq", 0) + + +def _as_baseline(path: str) -> ParameterTest: + return ParameterTest(path, "eq_path", f"baseline.{path}") + + +UK_ENGINE_POPULATION = PopulationDeclaration( + engine="policyengine_uk", + distribution="policyengine-uk", + reviewed_version="2.174.1", + list_source="policyengine_uk.utils.household_subset:WHOLE_POPULATION_VARIABLES", + weight_variable="household_weight", + corrected={ + # shareholding.py and corporate_land_value.py: corporate_sector_wealth + # over its weighted total (times the aggregate corporate land value). + "shareholding": "corporate_sector_wealth", + "corporate_land_value": "corporate_sector_wealth", + # consumption_share.py: consumption over its weighted total. + "consumption_shareholding": "consumption", + }, + key_adds={ + "corporate_sector_wealth": ("corporate_wealth", "private_pension_wealth"), + "consumption": _COICOP_CONSUMPTION, + }, + stored=("months_since_last_birthday", "attends_private_school"), + zero_at_policy={ + # Zero without both elasticities (its formula's first test). + "employment_income_behavioral_response": all_of( + _zero("gov.simulation.labour_supply_responses.income_elasticity"), + _zero("gov.simulation.labour_supply_responses.substitution_elasticity"), + ), + # Zero without the channel's incidence, or with all of it on employees. + "employer_ni_response_capital_incidence": any_of( + _zero(f"{_EMPLOYER_NI}.capital_incidence"), + ParameterTest(f"{_EMPLOYER_NI}.employee_incidence", "eq", 1), + ), + "employer_ni_response_consumer_incidence": any_of( + _zero(f"{_EMPLOYER_NI}.consumer_incidence"), + ParameterTest(f"{_EMPLOYER_NI}.employee_incidence", "eq", 1), + ), + # Both incidence terms carry the rate. + "carbon_tax": _zero("gov.contrib.ubi_center.carbon_tax.rate"), + "consumer_incident_tax_revenue_change": _zero( + f"{_BUDGET}.consumer_incident_tax_change" + ), + "education_budget_change": _zero(f"{_BUDGET}.education"), + "nhs_budget_change": _zero(f"{_BUDGET}.nhs"), + "other_public_spending_budget_change": _zero( + f"{_BUDGET}.other_public_spending" + ), + }, + inert={ + # Attendance reaches household taxes only through + # private_school_vat_change (household_tax.py), pupils times fees + # times the difference between private_school_vat_rate under the + # policy and under its baseline (private_school_vat.py); the rate + # reads these three parameters. gov_tax adds the level. + "attends_private_school": InertRule( + sink="private_school_vat_change", + levels=("private_school_vat", "baseline_private_school_vat"), + carriers=("gov_tax",), + predicate=all_of( + _as_baseline("gov.hmrc.vat.private_school_fees_exempt"), + _as_baseline("gov.hmrc.vat.standard_rate"), + _as_baseline("gov.contrib.labour.private_school_vat"), + ), + ), + }, + refused=( + "household_income_decile", + "household_income_decile_ahc", + "household_wealth_decile", + "in_relative_poverty_ahc", + "in_relative_poverty_bhc", + "pre_budget_change_ons_household_income_decile", + ), + window=PositionWindow( + first_year=2026, + source="policyengine_uk.scenarios.uc_reform:REBALANCING_YEARS", + reason=( + "policyengine-uk's constructor draws which Universal Credit " + "claimants are new claimants (the health element they receive) " + "with one random number per benefit unit in the order the " + "simulation holds them, for each rebalancing year from 2026, so " + "a block or batch draws other statuses than the pool" + ), + ), + # Recorded for the receipts: what reads the corrected shares. + observed=("LVT", "business_rates", "corporate_sdlt"), + # create_structural_reforms_from_parameters (reforms/reforms.py) applies + # these in the constructor, before any guard is armed; adjust_budgets + # would rescale spending by a population total. + constructor_factories=( + "policyengine_uk.reforms.cps.marriage_tax_reforms:create_marriage_tax_reform", + "policyengine_uk.reforms.conservatives.household_based_hitc:" + "create_household_based_hitc_reform", + "policyengine_uk.reforms.policyengine.disable_simulated_benefits:" + "disable_simulated_benefits", + "policyengine_uk.reforms.policyengine.adjust_budgets:adjust_budgets", + "policyengine_uk.reforms.scotland.scottish_child_payment_reform:" + "create_scottish_child_payment_reform", + ), + digests={ + "policyengine_uk/variables/household/wealth/shareholding.py": ( + "95fae830757b8db478cab62c1e84b093af268abd8d0ed21a518c65b0d635a52b" + ), + "policyengine_uk/variables/household/wealth/corporate_land_value.py": ( + "2082f7890365a40c20c68a7e2cbc7d9cccfeacb13ce0684670bb960e138bcae2" + ), + "policyengine_uk/variables/household/consumption/consumption_share.py": ( + "b2e59e6f3a32f33482630938e385c07762265da2cf9f0848cd011a6166ad30cf" + ), + "policyengine_uk/variables/contrib/ubi_center/carbon_tax.py": ( + "ce9ce9c9fe346c9b6fcaeec6cd03fed602c1168bbd691a0fca24c797f3235627" + ), + "policyengine_uk/variables/contrib/policyengine/" + "consumer_incident_tax_revenue_change.py": ( + "47dfa0d09317dd652bbbcecd46a76466d72eade2abaaf9a0a3bd7e5df9cf4304" + ), + "policyengine_uk/variables/contrib/policyengine/education_budget_change.py": ( + "4f1723792c35d41f3a13d1d790825bff830f5332fb2e134202ef8f266978d469" + ), + "policyengine_uk/variables/contrib/policyengine/nhs_budget_change.py": ( + "667a919cd07f0ad61c860473268770e7e76e77202e9f595c0e6e88711b5b6c48" + ), + "policyengine_uk/variables/contrib/policyengine/" + "other_public_spending_budget_change.py": ( + "3ce3ac97d8a473f60732fcefdf4e4e6430958493f7ae132f14555a641a5488f1" + ), + "policyengine_uk/variables/contrib/policyengine/employer_ni/" + "employer_ni_response_capital_incidence.py": ( + "6cb5bdd539d22342f0c614c8561ad0b86d45fdd545c3e814c819cbeff64512bd" + ), + "policyengine_uk/variables/contrib/policyengine/employer_ni/" + "employer_ni_response_consumer_incidence.py": ( + "abfef8b2f57cc489282b09634e1d96f7464a64040e1ed701f01cdc8d30268058" + ), + "policyengine_uk/variables/gov/simulation/labour_supply_response/" + "employment_income_behavioral_response.py": ( + "aeac3d6e861b742c2d260fe513ad430e4411cbe9804451076391fda9e4b64e07" + ), + "policyengine_uk/variables/contrib/labour/attends_private_school.py": ( + "1c0dc90e234cba789f31f94bdc7b619c882b155d63858939beda4ef8a7334c21" + ), + "policyengine_uk/variables/gov/hmrc/private_school_vat.py": ( + "389885eaead3ccc87971218fe5f6d5e8e50cdbb835b255096f0f82569e75cb30" + ), + "policyengine_uk/variables/gov/hmrc/baseline_private_school_vat.py": ( + "3ee5f683b7b9461203d166e3b7f0ccfa305b14819da3a474163a7d0060988269" + ), + "policyengine_uk/variables/gov/hmrc/private_school_vat_change.py": ( + "9d373dcd40dcb74747afd43d5637d7b650e65916992d31b34d08a7cba0b347ee" + ), + "policyengine_uk/reforms/policyengine/adjust_budgets.py": ( + "4aabd28d8bfdd617e4f29f5150e3ddc6cadd97ee5270b77455d0dd71c1acd23a" + ), + "policyengine_uk/reforms/reforms.py": ( + "51341a9f1494944eeea945bcc614f047de568bc455a00936f794d4b5984e42eb" + ), + "policyengine_uk.scenarios.uc_reform:add_universal_credit_reform": ( + "ca13079bb2043c8965dd2cc9397c13c58b7f98f8e58b2c678ab7602f72a55935" + ), + }, +) + + +def uk_engine_population(system: Any | None = None) -> PopulationDeclaration: + """The declaration, validated against the installed policyengine-uk. + + Pass the tax-benefit system the caller's simulations use; without one a + fresh ``CountryTaxBenefitSystem`` is built. Raises + :class:`~microcosm.frame.adapters.engine_population.PopulationDeclarationError` + when the engine has moved from what was reviewed, so per-clone blocks + and batches never run on an unreviewed engine. Only callers that block + or batch call this; a single-engine run never needs it. + """ + + try: + import policyengine_uk + except ImportError as exc: + raise ImportError( + "The UK whole-population declaration requires the 'policyengine-uk' " + "package. Install it with 'microcosm-frame[uk]'." + ) from exc + if system is None: + system = policyengine_uk.CountryTaxBenefitSystem() + validate_declaration(UK_ENGINE_POPULATION, system) + return UK_ENGINE_POPULATION diff --git a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_population.py b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_population.py new file mode 100644 index 0000000000..0fb009de66 --- /dev/null +++ b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_population.py @@ -0,0 +1,1083 @@ +"""policyengine-uk's whole-population formulas as the UK declares them (microcosm#931). + +``UK_ENGINE_POPULATION`` is held against the installed engine: its own list, +the allocation keys' inputs, the pinned sources and their coverage, the +predicates at current law and moved off it, the constructor's structural +reforms, the window, the readers the inert rule rests on, and the drift +lists of axis-free reductions and whole-array branches outside the list. +The shared machinery then runs on small synthetic datasets: a pool of two +clones that differ, corrected clone by clone, against one engine over the +pool. Every household here is made up. +""" + +from __future__ import annotations + +import ast +import dataclasses +import inspect +import math +import re +import subprocess +import sys +import textwrap +import types +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.frame.adapters.engine_population import ( + MEASURE_PASS, + FormulaRunRecorder, + PopulationDeclarationError, + check_key_sums, + check_uniform_growth, + inject_pool_shares, + judge, + pool_share_atol, + pool_share_rtol, + pool_shares, + source_sha256, + validate_declaration, +) +from microcosm.frame.adapters.policyengine_uk_population import ( + UK_ENGINE_POPULATION, + UK_SPINE_ENGINE_BATCH_COLUMN, + uk_engine_population, +) + +D, P = 2024, 2025 +CLONES, HOUSEHOLDS = 2, 30 +COICOP = UK_ENGINE_POPULATION.key_adds["consumption"] +REGIONS = ("LONDON", "NORTH_WEST", "SCOTLAND", "WALES", "SOUTH_EAST") + + +@pytest.fixture(scope="module") +def system(): + from policyengine_uk import CountryTaxBenefitSystem + + return CountryTaxBenefitSystem() + + +def _household_tax_variables(): + from policyengine_uk.variables.gov.hmrc.household_tax import ( + HOUSEHOLD_TAX_VARIABLES, + ) + + return HOUSEHOLD_TAX_VARIABLES + + +def _package_root() -> Path: + import policyengine_uk + + return Path(policyengine_uk.__file__).parent + + +# --------------------------------------------------------------------------- +# The declaration against the installed engine +# --------------------------------------------------------------------------- + + +def test_the_modules_load_without_importing_an_engine(): + code = ( + "import sys\n" + "import microcosm.frame.adapters.policyengine_uk_population\n" + "loaded = [m for m in sys.modules if m.startswith('policyengine')]\n" + "assert not loaded, loaded\n" + ) + subprocess.run([sys.executable, "-c", code], check=True) + + +def test_the_declaration_classifies_exactly_the_engines_list(): + from policyengine_uk.utils.household_subset import WHOLE_POPULATION_VARIABLES + + assert UK_ENGINE_POPULATION.classified() == WHOLE_POPULATION_VARIABLES + assert len(WHOLE_POPULATION_VARIABLES) == 19 + + +def test_the_declaration_validates_against_the_installed_engine(system): + assert uk_engine_population(system) is UK_ENGINE_POPULATION + + +def test_the_allocation_keys_are_the_engines_adds_of_household_inputs(system): + for formula, key in UK_ENGINE_POPULATION.corrected.items(): + variable = system.variables[formula] + assert not variable.is_input_variable(), formula + assert variable.entity.key == "household", formula + key_variable = system.variables[key] + assert tuple(key_variable.adds) == UK_ENGINE_POPULATION.key_adds[key] + for name in key_variable.adds: + leaf = system.variables[name] + assert leaf.is_input_variable() and leaf.entity.key == "household", name + assert UK_ENGINE_POPULATION.key_adds["corporate_sector_wealth"] == ( + "corporate_wealth", + "private_pension_wealth", + ) + assert len(COICOP) == 12 + + +@pytest.mark.parametrize("reference", sorted(UK_ENGINE_POPULATION.digests)) +def test_every_pinned_source_is_the_reviewed_one(reference): + assert source_sha256(reference) == UK_ENGINE_POPULATION.digests[reference] + + +def _source_file(system, name): + """The file a variable's class is defined in, as a digest names it.""" + + path = Path(inspect.getsourcefile(type(system.variables[name]))).resolve() + return f"policyengine_uk/{path.relative_to(_package_root().resolve()).as_posix()}" + + +def test_the_pins_cover_every_source_a_class_rests_on(system): + # Every corrected, zero-at-policy and inert formula, the inert rule's sink + # and levels, the constructor's structural reforms (adjust_budgets would + # rescale spending by a population total) and the Universal Credit + # modifier the window rests on; nothing else. + declaration = UK_ENGINE_POPULATION + names = {*declaration.corrected, *declaration.zero_at_policy} + for name, rule in declaration.inert.items(): + names |= {name, rule.sink, *rule.levels} + required = {_source_file(system, name) for name in names} | { + "policyengine_uk/reforms/policyengine/adjust_budgets.py", + "policyengine_uk/reforms/reforms.py", + "policyengine_uk.scenarios.uc_reform:add_universal_credit_reform", + } + assert set(declaration.digests) == required + + +def test_a_source_that_moved_fails_validation(system): + reference = "policyengine_uk/variables/household/wealth/shareholding.py" + moved = dataclasses.replace( + UK_ENGINE_POPULATION, + digests={**UK_ENGINE_POPULATION.digests, reference: "0" * 64}, + ) + with pytest.raises(PopulationDeclarationError, match="has changed"): + validate_declaration(moved, system) + + +@pytest.mark.parametrize("year", [D, P]) +def test_every_predicate_holds_at_current_law(system, year): + parameters = system.parameters(year) + for name, predicate in UK_ENGINE_POPULATION.run_predicates().items(): + assert predicate.holds(parameters), (name, year) + + +@pytest.mark.parametrize("year", [D, P]) +def test_the_inert_predicate_gives_the_engines_rate_equality(system, year): + from policyengine_uk.variables.gov.hmrc.private_school_vat import ( + private_school_vat_rate, + ) + + parameters = system.parameters(year) + rule = UK_ENGINE_POPULATION.inert["attends_private_school"] + assert rule.predicate.holds(parameters) + assert private_school_vat_rate(parameters.gov) == private_school_vat_rate( + parameters.baseline.gov + ) + # The rate reads exactly the three parameters the predicate compares. + source = inspect.getsource(private_school_vat_rate) + read = set(re.findall(r"(?:vat|gov)\.([a-z_.]+)", source)) + assert read == { + "hmrc.vat", + "private_school_fees_exempt", + "standard_rate", + "contrib.labour.private_school_vat", + } + + +_LSR = "gov.simulation.labour_supply_responses" +_EMPLOYER_NI = "gov.contrib.policyengine.employer_ni" +_BUDGET = "gov.contrib.policyengine.budget" + +#: For each formula with a predicate, moves of its guarded parameters off +#: current law, each of which makes the predicate false; together they move +#: every parameter it guards. +PREDICATE_MOVES = { + "employment_income_behavioral_response": ( + {f"{_LSR}.income_elasticity": 0.1}, + {f"{_LSR}.substitution_elasticity": 0.1}, + ), + "employer_ni_response_capital_incidence": ( + { + f"{_EMPLOYER_NI}.capital_incidence": 0.5, + f"{_EMPLOYER_NI}.employee_incidence": 0.5, + }, + ), + "employer_ni_response_consumer_incidence": ( + { + f"{_EMPLOYER_NI}.consumer_incidence": 0.5, + f"{_EMPLOYER_NI}.employee_incidence": 0.5, + }, + ), + "carbon_tax": ({"gov.contrib.ubi_center.carbon_tax.rate": 50.0},), + "consumer_incident_tax_revenue_change": ( + {f"{_BUDGET}.consumer_incident_tax_change": 1.0}, + ), + "education_budget_change": ({f"{_BUDGET}.education": 1.0},), + "nhs_budget_change": ({f"{_BUDGET}.nhs": 1.0},), + "other_public_spending_budget_change": ({f"{_BUDGET}.other_public_spending": 1.0},), + # Each moves the private school VAT rate off its baseline. + "attends_private_school": ( + {"gov.hmrc.vat.private_school_fees_exempt": True}, + {"gov.hmrc.vat.standard_rate": 0.25}, + {"gov.contrib.labour.private_school_vat": 0.3}, + ), +} +#: What else a formula needs, beyond its predicate failing, to have +#: something to respond to. +RESPONSE_NEEDS = { + # A change in net income and marginal rates. + "employment_income_behavioral_response": { + "gov.hmrc.income_tax.rates.uk[0].rate": 0.25 + }, + # A change in employers' costs. + "employer_ni_response_capital_incidence": { + "gov.hmrc.national_insurance.class_1.rates.employer": 0.2 + }, + "employer_ni_response_consumer_incidence": { + "gov.hmrc.national_insurance.class_1.rates.employer": 0.2 + }, +} + + +def test_every_guarded_parameter_is_moved(): + declaration = UK_ENGINE_POPULATION + predicates = declaration.run_predicates() + assert set(PREDICATE_MOVES) == {*declaration.zero_at_policy, *declaration.inert} + for name, moves in PREDICATE_MOVES.items(): + guarded = { + path + for path in predicates[name].paths() + if not path.startswith("baseline.") + } + assert {path for move in moves for path in move} == guarded, name + + +def _control_tables(): + """One clone of the synthetic households, its children at private + school, so the private school VAT change has pupils to charge.""" + + person, benunit, household = _subset(_tables(), 0) + person["attends_private_school"] = person["age"] < 18 + return person, benunit, household + + +@pytest.mark.parametrize( + "name, move", + [(name, move) for name, moves in PREDICATE_MOVES.items() for move in moves], + ids=lambda value: value if isinstance(value, str) else "+".join(sorted(value)), +) +def test_a_moved_parameter_breaks_its_predicate_and_frees_its_formula(name, move): + # The negative control for each predicate: off current law it fails, and + # the formula it guards (an inert formula's sink) is no longer zero. + rule = UK_ENGINE_POPULATION.inert.get(name) + changes = {**move, **RESPONSE_NEEDS.get(name, {})} + simulation = _simulation( + *_control_tables(), + reform={ + path: {"2024-01-01.2100-12-31": value} for path, value in changes.items() + }, + ) + predicate = UK_ENGINE_POPULATION.run_predicates()[name] + assert not predicate.holds(simulation.tax_benefit_system.parameters(P)) + target = name if rule is None else rule.sink + assert np.any(_values(simulation, target, P) != 0), target + + +@pytest.mark.parametrize("year", [D, P]) +def test_no_structural_reform_is_switched_on(system, year): + from policyengine_uk.reforms import reforms + + for reference in UK_ENGINE_POPULATION.constructor_factories: + module, _, name = reference.partition(":") + factory = getattr(__import__(module, fromlist=[name]), name) + assert factory(system.parameters, year) is None, reference + tree = ast.parse( + inspect.getsource(reforms.create_structural_reforms_from_parameters) + ) + called = { + node.func.id + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id.startswith(("create_", "disable_", "adjust_")) + } + declared = { + reference.rpartition(":")[2] + for reference in UK_ENGINE_POPULATION.constructor_factories + } + assert called == declared + + +def test_the_window_starts_at_the_rebalancing_years(): + from policyengine_uk.scenarios.uc_reform import ( + FIRST_REBALANCING_YEAR, + REBALANCING_YEARS, + ) + + assert REBALANCING_YEARS.start == FIRST_REBALANCING_YEAR == 2026 + assert UK_ENGINE_POPULATION.window.first_year == REBALANCING_YEARS.start + assert UK_SPINE_ENGINE_BATCH_COLUMN == "household_clone_index" + + +def _readers(names: set[str]) -> dict[str, set[str]]: + """Where each name appears as a string constant or is called, by the + module and outermost definition (outside the engine's own tests).""" + + root = _package_root() + found: dict[str, set[str]] = {name: set() for name in names} + for path in sorted(root.rglob("*.py")): + relative = path.relative_to(root).as_posix() + if relative.startswith("tests/"): + continue + tree = ast.parse(path.read_text(encoding="utf-8")) + for top in tree.body: + scope = getattr(top, "name", None) + if scope is None and isinstance(top, ast.Assign): + targets = [t.id for t in top.targets if isinstance(t, ast.Name)] + scope = targets[0] if targets else None + where = f"{relative}:{scope or ''}" + for node in ast.walk(top): + value = None + if isinstance(node, ast.Constant) and isinstance(node.value, str): + value = node.value + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + value = node.func.id + if value in found: + found[value].add(where) + return found + + +def test_attendance_reaches_household_taxes_only_through_the_vat_change(): + readers = _readers( + { + "attends_private_school", + "private_school_vat_liability", + "private_school_vat", + "baseline_private_school_vat", + "private_school_vat_change", + } + ) + assert readers["attends_private_school"] == { + "data/filter_dataset.py:filter_dataset", + "utils/household_subset.py:WHOLE_POPULATION_VARIABLES", + # A calibration script, run by hand; no formula imports it. + "utils/solve_private_school_attendance_factor.py:", + "variables/gov/hmrc/private_school_vat.py:private_school_vat_liability", + } + assert readers["private_school_vat_liability"] == { + "variables/gov/hmrc/baseline_private_school_vat.py:baseline_private_school_vat", + "variables/gov/hmrc/private_school_vat.py:private_school_vat", + } + assert readers["private_school_vat"] == { + "variables/gov/gov_tax.py:GOV_TAX_VARIABLES", + "variables/gov/hmrc/private_school_vat_change.py:private_school_vat_change", + } + assert readers["baseline_private_school_vat"] == { + "variables/gov/hmrc/private_school_vat_change.py:private_school_vat_change", + } + assert readers["private_school_vat_change"] == { + "variables/gov/hmrc/household_tax.py:HOUSEHOLD_TAX_VARIABLES", + } + + +#: Formulas outside the engine's list with an axis-free reduction (a sum, +#: mean, extreme or any/all over the whole array) in their own source, +#: reviewed as household-local: membership tests and shortcuts, a loop bound +#: and parameter-share cumulative sums. A new hit must be reviewed. +AXIS_FREE_REDUCTIONS = { + "brma": [".any()"], + "capital_gains_tax": ["np.any(", "np.max("], + "ebr_council_tax_rebate": ["np.any("], + "is_cared_for_by_carer_benefit_recipient": ["np.max("], + "legacy_means_test_uncounted_income_tax_without_earnings": [".any()"], + "pension_credit_income_tax_without_earnings": [".any()"], + "pension_credit_income_tax_without_employment_income": [".any()"], + "uc_award_carries_deductions": [".any()"], + "uc_deduction_combination": [".sum()", "np.cumsum("], + "uc_latent_deduction_rate": [".sum()", "np.cumsum("], + "uses_property_allowance": [".all()", ".any()"], + "uses_trading_allowance": [".any()"], +} +_REDUCTION = re.compile( + r"\.(sum|mean|max|min|any|all|median|std|var|cumsum|argsort|sort_values|rank" + r"|quantile|percentile)\(\s*\)" + r"|np\.(sum|mean|max|min|any|all|median|quantile|percentile|sort|argsort" + r"|cumsum|unique)\(" + r"|numpy\.(sum|mean|max|min|any|all|median|quantile|percentile|sort|argsort" + r"|cumsum|unique)\(" +) + + +def test_axis_free_reductions_outside_the_list_are_the_reviewed_ones(system): + listed = UK_ENGINE_POPULATION.classified() + hits: dict[str, set[str]] = {} + for name, variable in system.variables.items(): + if name in listed: + continue + for formula in variable.formulas.values(): + found = { + m.group(0) for m in _REDUCTION.finditer(inspect.getsource(formula)) + } + if found: + hits.setdefault(name, set()).update(found) + assert {name: sorted(found) for name, found in hits.items()} == AXIS_FREE_REDUCTIONS + + +#: What marks an if, while or conditional-expression test (or a +#: comprehension filter) as reading the whole array rather than a row: a +#: reduction, any/all, or a length or size. +_WHOLE_ARRAY_TEST = ( + ".any()", + ".all()", + "np.any(", + "np.all(", + "numpy.any(", + "numpy.all(", + "any(", + "all(", + "len(", + ".sum()", + "np.sum(", + ".max()", + ".min()", + "np.max(", + "np.min(", + ".mean()", + "np.mean(", + ".size", + ".shape", + "np.unique(", + "np.median(", + ".count", + ".nunique(", +) + + +def _engine_function(function, root): + """``path:qualname`` of a function defined in the engine (not its tests).""" + + code = getattr(function, "__code__", None) + if code is None: + return None + try: + relative = Path(code.co_filename).resolve().relative_to(root) + except ValueError: + return None + if relative.parts[0] == "tests": + return None + return f"{relative.as_posix()}:{function.__qualname__}" + + +def _code_names(code): + names = set(code.co_names) + for constant in code.co_consts: + if isinstance(constant, types.CodeType): + names |= _code_names(constant) + return names + + +def _unwrapped(value): + try: + return inspect.unwrap(value) + except ValueError: + return value + + +def _called(function, root): + """The engine functions ``function`` names: globals, and attributes of + the modules it names, unwrapped from their decorators.""" + + namespace = getattr(function, "__globals__", {}) + names = _code_names(function.__code__) + for name in sorted(names): + value = namespace.get(name) + candidates = [] + if isinstance(value, types.ModuleType): + for other in sorted(names): + attribute = getattr(value, other, None) + if callable(attribute): + candidates.append(_unwrapped(attribute)) + elif callable(value): + candidates.append(_unwrapped(value)) + for candidate in candidates: + if isinstance(candidate, types.FunctionType): + if _engine_function(candidate, root) is not None: + yield candidate + + +def _whole_array_tests(function): + """Each such test in ``function``'s source, as written, on one line.""" + + try: + source = textwrap.dedent(inspect.getsource(function)) + tree = ast.parse(source) + except (OSError, TypeError, SyntaxError): + return [] + found = set() + for node in ast.walk(tree): + conditions = [] + if isinstance(node, (ast.If, ast.While, ast.IfExp)): + conditions.append(node.test) + elif isinstance(node, ast.comprehension): + conditions.extend(node.ifs) + for condition in conditions: + text = " ".join(ast.get_source_segment(source, condition).split()) + if any(mark in text for mark in _WHOLE_ARRAY_TEST): + found.add(text) + return sorted(found) + + +def whole_array_branches(system): + """Every formula outside the engine's list, and every engine function + those formulas reach, with its whole-array tests.""" + + root = _package_root().resolve() + listed = UK_ENGINE_POPULATION.classified() + reached = {} + for name, variable in system.variables.items(): + if name in listed: + continue + stack = list(variable.formulas.values()) + while stack: + function = stack.pop() + label = _engine_function(function, root) + if label is None or label in reached: + continue + reached[label] = function + stack.extend(_called(function, root)) + branches = {} + for label, function in sorted(reached.items()): + tests = _whole_array_tests(function) + if tests: + branches[label] = tests + return branches + + +#: Formulas outside the engine's list, and the engine functions they reach, +#: whose control flow turns on the whole array. Each was reviewed as +#: household-local, so a block takes each household's branch as the pool +#: does (microcosm#931): a shortcut whose branches give every row the same +#: value, a test of parameters or of which inputs the data supplies, a +#: check that raises, or policyengine-uk's own household-subset machinery, +#: which falls back to the whole simulation. A new or changed test must be +#: reviewed the same way. +WHOLE_ARRAY_BRANCHES = { + # A configuration check that raises. + "data/isa_income.py:validate_contract": [ + 'not all( isinstance(contract[key], str) for key in ("representation", "projection") )', + ], + # Where every couple is the claimant and partner, np.where gives each the + # same award. + "utils/benefit_unit.py:award_of_couple": [ + "np.all(same_couple)", + ], + # Raises where the baseline holds other people. + "utils/capital_gains.py:measure_capital_gains_mtrs": [ + "baseline_count != person.count", + ], + # Without gains nobody's gains respond, as the factors would give. + "utils/capital_gains.py:realisation_factors": [ + 'not np.any(person("capital_gains_before_response", period) != 0)', + ], + # Without participation the working terms are false; targeted eligibility + # counts only where a row's own terms overlap. + "utils/childcare.py:childcare_term_value": [ + 'component in ("targeted", "extended") or ( component == "universal" and any(np.any(u & t) for u, t in zip(universal_terms, targeted_terms)) )', + "np.any(participating)", + ], + # Checks that raise. + "utils/dates.py:_check_given_dates_of_birth": [ + "invalid.any()", + "mismatch.any()", + ], + # Checks that raise; the instant is chosen row by row. + "utils/dates.py:birth_instant": [ + "(given < 0).any()", + "use_given.any()", + ], + # policyengine-uk's household subsets, which fall back to the whole + # simulation. + "utils/household_subset.py:_positions": [ + "(kept[members] != kept_people).any()", + ], + "utils/household_subset.py:_restrict": [ + "np.ndim(values) != 1 or len(values) != population.count", + ], + "utils/household_subset.py:_restricted_branch": [ + "any( baseline.populations[key].count != population.count or not np.array_equal(baseline.populations[key].ids, population.ids) for key, population in simulation.populations.items() )", + "any( holder._disk_storage is not None for population in simulation.populations.values() for holder in population._holders.values() )", + ], + "utils/household_subset.py:household_subset": [ + "not households.all()", + ], + # A parameter scale. + "utils/income_tax.py:first_extended_scottish_threshold": [ + "len(scale.thresholds) == 3", + ], + # Rows without net pay and a trading loss take net income either way (net + # income is floored at zero). + "utils/pensions.py:annual_allowance_net_income": [ + 'not np.any((relievable_net_pay > 0) & (person("trading_loss", period) > 0))', + ], + # Parameter trees. + "utils/rule_comparison.py:same_parameters": [ + 'len(node.brackets) != len(other_node.brackets) or node.metadata.get( "type" ) != other_node.metadata.get("type")', + "len(values) != len(other_values)", + ], + # Which inputs the data supplies. + "variables/gov/dft/person_bus_fare_spending.py:person_bus_fare_spending.formula": [ + 'all( supplied_input(person, name, period) is None for name in ("bus_in_london_trips", "other_local_bus_trips") )', + ], + # Where no household is over its total, np.where keeps every balance. + "variables/gov/dwp/_person_capital_pool.py:bounded_holdings": [ + "not over.any()", + ], + # Parts of the year from parameters, and whether an input or reform + # supplies the award. + "variables/gov/dwp/carers_allowance.py:carers_allowance.formula": [ + "len(parts) == 1 or not calculated_by_model( person, period, carers_allowance_pre_overlap )", + ], + # Parts of the year from parameters. + "variables/gov/dwp/carers_allowance_pre_overlap.py:carers_allowance_pre_overlap.formula": [ + "len(parts) == 1", + ], + # Segments of the year from parameters, and whether an input overrides the + # income. + "variables/gov/dwp/housing_benefit/entitlement/housing_benefit_entitlement.py:housing_benefit_entitlement.formula": [ + 'len(segments) == 1 or hb_uses_annual_override( benunit, period, "housing_benefit_applicable_income", housing_benefit_applicable_income.formula, )', + ], + # Zeroing earnings nobody has changes nobody's tax. + "variables/gov/dwp/legacy_means_test_uncounted_income_tax_without_earnings.py:legacy_means_test_uncounted_income_tax_without_earnings.formula": [ + "not any((person(source, period) != 0).any() for source in sources)", + ], + # A parameter list. + "variables/gov/dwp/pension_credit/pension_credit_earnings_disregard.py:pension_credit_earnings_disregard.formula": [ + "len(unit_disability_benefits) > 0", + ], + # Zeroing earnings nobody has changes nobody's tax. + "variables/gov/dwp/pension_credit/pension_credit_earnings_income_tax.py:pension_credit_income_tax_without_earnings.formula": [ + 'not (person("self_employment_income", period) != 0).any()', + ], + "variables/gov/dwp/pension_credit/pension_credit_earnings_income_tax.py:pension_credit_income_tax_without_employment_income.formula": [ + "not any((person(source, period) != 0).any() for source in sources)", + ], + # Without a changed baseline or an award, every unit carries its + # deductions, as the comparison gives a unit without an award. + "variables/gov/dwp/universal_credit/deductions/uc_award_carries_deductions.py:uc_award_carries_deductions.formula": [ + 'not getattr(simulation, "changed_from_baseline", False) or baseline is None or not award.any() or baseline.populations["person"].count != person.count', + ], + # A parameter list. + "variables/gov/dwp/universal_credit/forgoes_universal_credit_for_tax_free_childcare.py:forgoes_universal_credit_for_tax_free_childcare.formula": [ + "not any( program in UNIVERSAL_CREDIT_AWARDS for program in p.disqualifying_benefits )", + ], + # With no exemption left anywhere, a further pass takes nothing. + "variables/gov/hmrc/capital_gains_tax/capital_gains_tax.py:capital_gains_tax.formula": [ + "not np.any(aea_left > 0)", + ], + # Each household decides its own route, and households do not move each + # other's tax; the tests skip work nobody needs. + "variables/gov/hmrc/income_tax/allowances/uses_property_allowance.py:uses_property_allowance.formula": [ + "(deciding & ~unsettled).any()", + "(unsettled & (most_charge_not_using > 0)).any()", + "(unsettled & (most_charge_using > 0)).any()", + "not choosing.any()", + "not done.all()", + ], + # Only a claimant can take the allowance. + "variables/gov/hmrc/income_tax/allowances/uses_trading_allowance.py:uses_trading_allowance.formula": [ + "not claimant.any()", + ], + # Parameter thresholds. + "variables/gov/hmrc/pensions/private_pension_contributions_tax.py:_top_slice_charge": [ + "i + 1 < len(thresholds)", + ], + # A parameter list. + "variables/gov/ofgem/warm_home_discount.py:warm_home_discount.formula": [ + "len(benefits) == 0", + ], + # A household that has not moved keeps its BRMA either way. + "variables/household/BRMA.py:brma.formula": [ + "not moved.any()", + ], +} + + +def test_whole_array_branches_outside_the_list_are_the_reviewed_ones(system): + assert whole_array_branches(system) == WHOLE_ARRAY_BRANCHES + + +# --------------------------------------------------------------------------- +# The machinery on the engine +# --------------------------------------------------------------------------- + + +def _tables(*, differ: bool = True): + """Two clones of the same households; with ``differ`` each clone draws + its own keys, pay and weights. Household ids rise with the clone.""" + + persons, benunits, households = [], [], [] + for clone in range(CLONES): + for h in range(HOUSEHOLDS): + household_id = (clone + 1) * 1000 + h + draw = np.random.default_rng(17 * h + (1009 * clone if differ else 0)) + shared = np.random.default_rng(5000 + h) + benunits.append({"benunit_id": household_id * 10}) + weight = 40_000.0 * (1 + h % 5) / CLONES + if differ: + weight *= float(draw.uniform(0.7, 1.3)) + row = { + "household_id": household_id, + "household_weight": weight, + "region": REGIONS[h % len(REGIONS)], + "council_tax": 1_400.0 + 90 * (h % 5), + "rent": 9_000.0 if h % 2 else 0.0, + "tenure_type": "RENT_PRIVATELY" if h % 2 else "OWNED_OUTRIGHT", + "corporate_wealth": float(draw.lognormal(10, 1.6)) if h % 2 else 0.0, + "private_pension_wealth": ( + float(draw.lognormal(11, 1.1)) if h % 3 else 0.0 + ), + } + for column in COICOP: + row[column] = float(draw.uniform(300, 6_000)) + households.append(row) + for p in range(1 + h % 4): + age = (41, 38, 12, 9)[p] + pay = np.random.default_rng(31 * h + p + (997 * clone if differ else 0)) + persons.append( + { + "person_id": household_id * 10 + p, + "person_benunit_id": household_id * 10, + "person_household_id": household_id, + "age": age, + "gender": "MALE" if (h + p) % 2 else "FEMALE", + "employment_income": ( + float(pay.uniform(0, 60_000)) + if age >= 18 and h % 3 + else 0.0 + ), + "months_since_last_birthday": float(shared.integers(0, 12)), + "attends_private_school_random_draw": float(shared.uniform()), + "attends_private_school": False, + } + ) + return pd.DataFrame(persons), pd.DataFrame(benunits), pd.DataFrame(households) + + +def _subset(tables, clone, *, weight_scale=1.0): + person, benunit, household = tables + ids = household["household_id"].to_numpy() + kept = household[(ids // 1000 - 1) == clone].copy() + kept["household_weight"] = kept["household_weight"] * weight_scale + people = person[person["person_household_id"].isin(set(kept["household_id"]))] + units = benunit[benunit["benunit_id"].isin(set(people["person_benunit_id"]))] + return people.copy(), units.copy(), kept + + +def _simulation(person, benunit, household, reform=None): + from policyengine_uk import Microsimulation + from policyengine_uk.data import UKSingleYearDataset + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return Microsimulation( + dataset=UKSingleYearDataset( + person=person.copy(), + benunit=benunit.copy(), + household=household.copy(), + fiscal_year=D, + ), + reform=reform, + ) + + +def _values(simulation, name, period, map_to=None): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + raw = simulation.calculate(name, period, map_to=map_to) + return np.asarray(raw) + + +def _key_sum(household, formula): + key = UK_ENGINE_POPULATION.corrected[formula] + values = np.zeros(len(household)) + for column in UK_ENGINE_POPULATION.key_adds[key]: + values += household[column].to_numpy(dtype=np.float64) + return float(np.sum(values * household["household_weight"].to_numpy(np.float64))) + + +#: Each share result and the allocation key it divides by: business rates +#: and corporate SDLT are shareholding times a receipt. +SHARE_RESULTS = { + "shareholding": "corporate_sector_wealth", + "corporate_land_value": "corporate_sector_wealth", + "consumption_shareholding": "consumption", + "business_rates": "corporate_sector_wealth", + "corporate_sdlt": "corporate_sector_wealth", +} +#: household_tax's share-dependent terms (the rest are zero at current law +#: or household-local). +SHARE_DEPENDENT_TAXES = ("business_rates", "corporate_sdlt") +HOUSEHOLD_LOCAL = ("income_tax", "national_insurance", "household_market_income") + + +def _share_rtol(name, block_rows, pool_rows, years): + key = SHARE_RESULTS[name] + return pool_share_rtol( + block_rows, + pool_rows, + key_inputs=len(UK_ENGINE_POPULATION.key_adds[key]), + projection_years=years, + ) + + +@pytest.fixture(scope="module") +def pool_engine(): + tables = _tables() + return tables, _simulation(*tables) + + +def test_zero_formulas_are_zero_on_a_synthetic_dataset(pool_engine): + _tables_, simulation = pool_engine + for name in UK_ENGINE_POPULATION.zero_at_policy: + for period in (D, P): + assert np.all(_values(simulation, name, period) == 0), (name, period) + + +def test_corrected_blocks_reproduce_one_engine_over_the_pool(pool_engine, system): + tables, pool = pool_engine + household = tables[2] + household_taxes = _household_tax_variables() + declaration = uk_engine_population(system) + clone_of = household["household_id"].to_numpy() // 1000 - 1 + pool_mass = float(household["household_weight"].sum()) + blocks = {clone: household[clone_of == clone] for clone in range(CLONES)} + scales = { + clone: pool_mass / float(rows["household_weight"].sum()) + for clone, rows in blocks.items() + } + shares = pool_shares( + { + clone: { + formula: _key_sum(rows, formula) for formula in declaration.corrected + } + for clone, rows in blocks.items() + }, + scales, + key_variables=declaration.corrected, + ) + pool_ids = _values(pool, "household_id", P) + pool_people = _values(pool, "person_id", P) + for clone, rows in blocks.items(): + block = _simulation(*_subset(tables, clone, weight_scale=scales[clone])) + check_key_sums( + block, + shares[clone], + D, + weight_variable=declaration.weight_variable, + ) + factors = check_uniform_growth( + block, + (*sorted(set(declaration.corrected.values())), declaration.weight_variable), + D, + P, + ) + assert all(factor is not None for factor in factors.values()) + here = np.searchsorted(pool_ids, _values(block, "household_id", P)) + for formula in declaration.corrected: + # Uncorrected, the block's own shares miss the pool's. + assert not np.allclose( + _values(block, formula, P), + _values(pool, formula, P)[here], + rtol=1e-3, + atol=0, + ), formula + injected = inject_pool_shares( + block, shares[clone], (D, P), weight_variable=declaration.weight_variable + ) + recorder = FormulaRunRecorder.for_declaration(declaration) + recorder.arm(block) + assert recorder.is_armed(block) and recorder.is_armed(block.baseline) + + requested = [(name, P) for name in (*SHARE_RESULTS, "household_tax")] + for name, period in requested: + _values(block, name, period) + built = [block] + if clone == 0: + # Another simulation the block builds from the same data (as the + # resolver's annual-income sibling is) takes the same arrays. + sibling = _simulation(*_subset(tables, clone, weight_scale=scales[clone])) + injected.apply(sibling) + recorder.arm(sibling) + for name in SHARE_RESULTS: + np.testing.assert_array_equal( + _values(sibling, name, P), _values(block, name, P), err_msg=name + ) + assert injected.matches(block) and injected.matches(sibling) + built.append(sibling) + for period, years in ((D, 0), (P, 1)): + for name in SHARE_RESULTS: + np.testing.assert_allclose( + _values(block, name, period), + _values(pool, name, period)[here], + rtol=_share_rtol(name, len(rows), len(household), years), + atol=0, + err_msg=f"{name} at {period}", + ) + # household_tax adds the share-dependent taxes to household-local + # ones: bounded per row, by the share terms and its own additions. + terms = { + name: _values(block, name, period, map_to="household") + for name in household_taxes + } + atol = pool_share_atol( + sum(np.abs(terms[name]) for name in SHARE_DEPENDENT_TAXES), + sum(np.abs(values.astype(np.float64)) for values in terms.values()), + share_rtol=max( + _share_rtol(name, len(rows), len(household), years) + for name in SHARE_DEPENDENT_TAXES + ), + additions=len(household_taxes), + ) + gap = np.abs( + _values(block, "household_tax", period).astype(np.float64) + - _values(pool, "household_tax", period)[here] + ) + assert np.all(gap <= atol), f"household_tax at {period}" + assert np.any(terms["business_rates"] != 0) + people = np.searchsorted(pool_people, _values(block, "person_id", P)) + for name in HOUSEHOLD_LOCAL: + values = _values(block, name, P) + reference = _values(pool, name, P) + positions = people if len(values) == len(people) else here + np.testing.assert_array_equal(values, reference[positions], err_msg=name) + + verdict = judge(declaration, MEASURE_PASS, recorder, built, (D, P), requested) + assert verdict.clean, verdict.describe() + assert verdict.classes["corrected"] == tuple(sorted(declaration.corrected)) + assert verdict.classes["stored"] == ( + "attends_private_school", + "months_since_last_birthday", + ) + assert "carbon_tax" in verdict.classes["zero_at_policy"] + + branch = block.get_branch("after_arming") + assert branch._run_formula.__self__ is branch + # Nothing the block computed: the branch runs it and records. + _values(branch, "household_net_income", D) + assert recorder.token(branch) is not None + + # A household-local read in a rebalancing year still breaks the window. + _values(block, "income_tax", 2026) + assert "income_tax" in recorder.periods_run()["2026"] + late = judge(declaration, MEASURE_PASS, recorder, built, (D, P), requested) + assert not late.clean + assert {(v.kind, v.period) for v in late.violations} == {("window_run", "2026")} + recorder.disarm() + assert all("_run_formula" not in vars(simulation) for simulation in built) + del block, branch, built, recorder, injected + + +def test_a_share_read_at_a_period_never_injected_is_caught(pool_engine, system): + tables, _pool = pool_engine + declaration = uk_engine_population(system) + block = _simulation(*_subset(tables, 1)) + household = _subset(tables, 1)[2] + shares = pool_shares( + { + 0: { + formula: _key_sum(household, formula) + for formula in declaration.corrected + }, + 1: { + formula: _key_sum(household, formula) + for formula in declaration.corrected + }, + }, + key_variables=declaration.corrected, + ) + inject_pool_shares(block, shares[1], (P,), weight_variable="household_weight") + recorder = FormulaRunRecorder.for_declaration(declaration) + recorder.arm(block) + _values(block, "business_rates", D) + verdict = judge(declaration, MEASURE_PASS, recorder, [block], (D, P)) + kinds = {(v.kind, v.name, v.period) for v in verdict.violations} + assert ("corrected_derived", "shareholding", str(D)) in kinds + assert ("corrected_not_input", "shareholding", str(D)) in kinds + recorder.disarm() + + +def test_a_twelve_input_key_that_rounds_apart_stays_inside_the_bound(): + # Two made-up one-person households. The engine adds each household's + # twelve COICOP inputs in float32: the block's eleven small ones each + # round down when added to the first, the rest of the pool's each round + # up. c_b comes from float64 frame sums, as the measure step forms it, so + # the adds' roundings reach the corrected share and need not cancel. + u = 2.0**-24 + small = { + 1000: float(np.float32(u * (1 - 2.0**-23))), + 2000: float(np.float32(u * (1 + 2.0**-22))), + } + weights = {1000: 1.0, 2000: 1023.0} + rows = [] + for household_id in (1000, 2000): + row = { + "household_id": household_id, + "household_weight": weights[household_id], + "region": "LONDON", + "council_tax": 1_400.0, + "rent": 0.0, + "tenure_type": "OWNED_OUTRIGHT", + "corporate_wealth": 0.0, + "private_pension_wealth": 0.0, + } + for index, column in enumerate(COICOP): + row[column] = 1024.0 * (1.0 if index == 0 else small[household_id]) + rows.append(row) + household = pd.DataFrame(rows) + person = pd.DataFrame( + [ + { + "person_id": household_id * 10, + "person_benunit_id": household_id * 10, + "person_household_id": household_id, + "age": 40, + "gender": "FEMALE", + "employment_income": 0.0, + "months_since_last_birthday": 6.0, + "attends_private_school_random_draw": 0.5, + "attends_private_school": False, + } + for household_id in (1000, 2000) + ] + ) + benunit = pd.DataFrame({"benunit_id": [10000, 20000]}) + sums = { + household_id: math.fsum( + float(value) + for value in household.loc[household["household_id"] == household_id].iloc[ + 0 + ][list(COICOP)] + ) + * weights[household_id] + for household_id in (1000, 2000) + } + scale = sum(weights.values()) / weights[1000] + shares = pool_shares( + {block: {"consumption_shareholding": total} for block, total in sums.items()}, + {block: sum(weights.values()) / weight for block, weight in weights.items()}, + key_variables={"consumption_shareholding": "consumption"}, + ) + + pool = _simulation(person, benunit, household) + alone = household[household["household_id"] == 1000].copy() + alone["household_weight"] = alone["household_weight"] * scale + block = _simulation(person.iloc[:1], benunit.iloc[:1], alone) + inject_pool_shares(block, shares[1000], (D,), weight_variable="household_weight") + corrected = float(_values(block, "consumption_shareholding", D)[0]) + reference = float(_values(pool, "consumption_shareholding", D)[0]) + gap = abs(corrected / reference - 1) + assert gap > pool_share_rtol(1, 2, key_inputs=1) + assert gap <= pool_share_rtol(1, 2, key_inputs=len(COICOP)) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py b/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py new file mode 100644 index 0000000000..72fc319103 --- /dev/null +++ b/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py @@ -0,0 +1,1744 @@ +"""The shared engine-block machinery on fake engines (microcosm#931). + +A fake simulation stands in for a policyengine-core one: holders that know +which periods hold inputs, formulas run through ``_run_formula``, a +parameter tree read at a period, and a ``clone`` that rebinds the methods +bound to the original, as core's does. Everything is synthetic. +""" + +from __future__ import annotations + +import copy +import math +import sys +import types +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.frame.adapters.engine_population import ( + FLOAT32_UNIT_ROUNDOFF, + MEASURE_PASS, + SPINE_BATCHED_PASS, + FormulaRunRecorder, + InertRule, + ParameterTest, + PassPolicy, + PoolShare, + PoolShareError, + PopulationDeclaration, + PopulationDeclarationError, + PopulationGuardError, + PositionWindow, + all_of, + any_of, + assert_groups_nest_in_households, + check_key_sums, + check_uniform_growth, + derived_watched_periods, + factor_spread, + float32_sum_roundings, + household_batches, + inject_pool_shares, + judge, + pool_share_atol, + pool_share_rtol, + pool_shares, + scatter_by_entity_ids, + source_sha256, + validate_declaration, +) + +D, P = 2024, 2025 +N = 6 +FAKE_ENGINE = "fake_population_engine" + + +# --------------------------------------------------------------------------- +# A fake engine +# --------------------------------------------------------------------------- + + +def _namespace(tree): + if isinstance(tree, dict): + return SimpleNamespace( + **{key: _namespace(value) for key, value in tree.items()} + ) + return tree + + +class FakeParameters: + """A parameter tree: called with a period it gives that period's values, + and its root's children are reachable as attributes, as core's are.""" + + def __init__(self, tree, overrides=None): + self.tree = tree + self.overrides = overrides or {} + + def __call__(self, period): + tree = copy.deepcopy(self.tree) + for path, value in self.overrides.get(int(str(period)[:4]), {}).items(): + *parents, leaf = path.split(".") + node = tree + for parent in parents: + node = node[parent] + node[leaf] = value + return _namespace(tree) + + def __getattr__(self, name): + tree = self.__dict__["tree"] + if name in tree: + return _namespace(tree[name]) + raise AttributeError(name) + + +class FakeVariable: + def __init__(self, name, *, formula=None, adds=None, dtype=np.float32): + self.name = name + self.entity = SimpleNamespace(key="household") + self.formula = formula + self.formulas = {"0000-01-01": formula} if formula is not None else {} + self.adds = adds + self.subtracts = None + self.dtype = dtype + + +class FakeHolder: + def __init__(self, variable): + self.variable = variable + self.arrays: dict[str, np.ndarray] = {} + self.inputs: set[str] = set() + + def get_known_periods(self): + return list(self.arrays) + + def get_input_periods(self, branch_name="default"): + return [period for period in self.arrays if period in self.inputs] + + def get_array(self, period, branch_name="default"): + return self.arrays.get(str(period)) + + def copy(self): + holder = FakeHolder(self.variable) + holder.arrays = {key: value.copy() for key, value in self.arrays.items()} + holder.inputs = set(self.inputs) + return holder + + +class FakeSimulation: + def __init__(self, system, inputs, *, rows=N, baseline=True): + self.tax_benefit_system = system + self.rows = rows + self.branch_name = "default" + self.branches = {} + self.holders = { + name: FakeHolder(variable) for name, variable in system.variables.items() + } + for (name, period), values in inputs.items(): + self.set_input(name, period, values) + if baseline: + self.baseline = self.clone() + + def get_holder(self, name): + return self.holders[name] + + def set_input(self, name, period, values): + holder = self.holders[name] + holder.arrays[str(period)] = np.array(values, dtype=holder.variable.dtype) + holder.inputs.add(str(period)) + + def calculate(self, name, period): + holder = self.holders[name] + cached = holder.arrays.get(str(period)) + if cached is not None: + return cached + variable = holder.variable + result = self._run_formula(variable, None, period) + if result is None: + values = np.zeros(self.rows, dtype=variable.dtype) + else: + values = np.broadcast_to( + np.asarray(result).astype(variable.dtype), (self.rows,) + ).copy() + holder.arrays[str(period)] = values + return values + + def _run_formula(self, variable, population, period): + if variable.formula is not None: + return variable.formula(self, period) + if variable.adds: + total = 0 + for name in variable.adds: + total = total + self.calculate(name, period) + return total + return None + + def clone(self): + """Copy as policyengine-core does: every attribute but the branches, + with the methods bound to the original bound to the copy.""" + + new = object.__new__(type(self)) + for key, value in self.__dict__.items(): + if key != "branches": + new.__dict__[key] = value + for key, value in list(new.__dict__.items()): + if isinstance(value, types.MethodType) and value.__self__ is self: + new.__dict__[key] = types.MethodType(value.__func__, new) + new.holders = {name: holder.copy() for name, holder in self.holders.items()} + new.branches = {} + return new + + def get_branch(self, name): + if name not in self.branches: + branch = self.clone() + branch.branch_name = name + branch.parent_branch = self + self.branches[name] = branch + return self.branches[name] + + +def _share(simulation, period): + key = simulation.calculate("key", period).astype(np.float64) + weight = simulation.calculate("weight", period).astype(np.float64) + total = float(np.sum(key * weight)) + return key / total if total else np.zeros_like(key) + + +def _rates(simulation, period): + return simulation.calculate("share", period) * 1_000.0 + + +def _budget_change(simulation, period): + budget = simulation.tax_benefit_system.parameters(period).gov.budget + if budget == 0: + return 0 + weight = simulation.calculate("weight", period) + return budget / weight.sum() + 0 * weight + + +def _months(simulation, period): + return np.full(simulation.rows, 6.0) + + +def _attendance(simulation, period): + income = simulation.calculate("local", period) + return income > np.median(income) + + +def _vat(rate_path): + def formula(simulation, period): + node = simulation.tax_benefit_system.parameters(period) + for part in rate_path.split("."): + node = getattr(node, part) + return simulation.calculate("attendance", period).astype(np.float64) * node + + return formula + + +def _vat_change(simulation, period): + return simulation.calculate("vat", period) - simulation.calculate( + "baseline_vat", period + ) + + +def _tax_total(simulation, period): + return simulation.calculate("vat", period) + simulation.calculate("rates", period) + + +def _decile(simulation, period): + income = simulation.calculate("local", period) + return np.argsort(np.argsort(income)) % 10 + + +def _local(simulation, period): + return simulation.calculate("key_a", period) * 2.0 + + +def _fake_system(*, budget=0.0, vat_rate=0.2, baseline_vat_rate=0.2): + names = { + "weight": FakeVariable("weight"), + "key_a": FakeVariable("key_a"), + "key_b": FakeVariable("key_b"), + "key": FakeVariable("key", adds=["key_a", "key_b"]), + "share": FakeVariable("share", formula=_share), + "rates": FakeVariable("rates", formula=_rates), + "months": FakeVariable("months", formula=_months), + "attendance": FakeVariable("attendance", formula=_attendance, dtype=bool), + "vat": FakeVariable("vat", formula=_vat("gov.vat_rate")), + "baseline_vat": FakeVariable( + "baseline_vat", formula=_vat("baseline.gov.vat_rate") + ), + "vat_change": FakeVariable("vat_change", formula=_vat_change), + "tax_total": FakeVariable("tax_total", formula=_tax_total), + "budget_change": FakeVariable("budget_change", formula=_budget_change), + "decile": FakeVariable("decile", formula=_decile, dtype=np.int64), + "local": FakeVariable("local", formula=_local), + } + tree = { + "gov": {"budget": budget, "vat_rate": vat_rate}, + "baseline": {"gov": {"vat_rate": baseline_vat_rate}}, + } + return SimpleNamespace(variables=names, parameters=FakeParameters(tree)) + + +def _quiet_reform(parameters, period): + return None + + +def _loud_reform(parameters, period): + return object() + + +def _late_reform(parameters, period): + return object() if int(str(period)[:4]) == P else None + + +@pytest.fixture +def fake_engine(monkeypatch): + """The engine's own module: its list, its window and its reform factories.""" + + module = types.ModuleType(FAKE_ENGINE) + module.WHOLE_POPULATION = frozenset( + {"share", "months", "attendance", "budget_change", "decile"} + ) + module.WINDOW = range(2026, 2033) + module.quiet_reform = _quiet_reform + module.loud_reform = _loud_reform + module.late_reform = _late_reform + monkeypatch.setitem(sys.modules, FAKE_ENGINE, module) + return module + + +def _declaration(**changes): + fields = { + "engine": FAKE_ENGINE, + "distribution": "fake-population-engine", + "reviewed_version": "1.0.0", + "list_source": f"{FAKE_ENGINE}:WHOLE_POPULATION", + "weight_variable": "weight", + "corrected": {"share": "key"}, + "key_adds": {"key": ("key_a", "key_b")}, + "stored": ("months", "attendance"), + "zero_at_policy": {"budget_change": ParameterTest("gov.budget", "eq", 0)}, + "inert": { + "attendance": InertRule( + sink="vat_change", + levels=("vat", "baseline_vat"), + carriers=("tax_total",), + predicate=all_of( + ParameterTest("gov.vat_rate", "eq_path", "baseline.gov.vat_rate") + ), + ) + }, + "refused": ("decile",), + "window": PositionWindow(2026, f"{FAKE_ENGINE}:WINDOW", "positional draws"), + "observed": ("rates",), + "constructor_factories": (f"{FAKE_ENGINE}:quiet_reform",), + } + fields.update(changes) + return PopulationDeclaration(**fields) + + +# Two clones of three households each: the first block's keys are larger. +KEY_A = np.array([10.0, 0.0, 30.0, 1.0, 2.0, 3.0]) +KEY_B = np.array([5.0, 5.0, 0.0, 1.0, 0.0, 2.0]) +WEIGHT = np.array([100.0, 200.0, 300.0, 150.0, 150.0, 150.0]) +BLOCKS = {"0": np.array([0, 1, 2]), "1": np.array([3, 4, 5])} +GROWTH = 1.03 + + +def _inputs(rows, *, weight_scale=1.0, store=("months", "attendance")): + inputs = {} + for period, factor in ((D, 1.0), (P, GROWTH)): + inputs[("key_a", period)] = KEY_A[rows] * factor + inputs[("key_b", period)] = KEY_B[rows] * factor + inputs[("weight", period)] = WEIGHT[rows] * weight_scale + if "months" in store: + inputs[("months", period)] = np.full(len(rows), 3.0) + if "attendance" in store: + inputs[("attendance", period)] = np.zeros(len(rows), dtype=bool) + return inputs + + +def _frame_sum(rows): + return float(np.sum((KEY_A[rows] + KEY_B[rows]) * WEIGHT[rows])) + + +def _block_shares(*, scaled): + pool_mass = float(WEIGHT.sum()) + scales = ( + {block: pool_mass / float(WEIGHT[rows].sum()) for block, rows in BLOCKS.items()} + if scaled + else None + ) + shares = pool_shares( + {block: {"share": _frame_sum(rows)} for block, rows in BLOCKS.items()}, + scales, + key_variables={"share": "key"}, + ) + return shares, scales + + +def _block(block, *, scaled=True, store=("months", "attendance"), system=None): + rows = BLOCKS[block] + scale = 1.0 + if scaled: + scale = float(WEIGHT.sum()) / float(WEIGHT[rows].sum()) + return FakeSimulation( + system or _fake_system(), + _inputs(rows, weight_scale=scale, store=store), + rows=len(rows), + ) + + +def _pool(): + return FakeSimulation(_fake_system(), _inputs(np.arange(N))) + + +# --------------------------------------------------------------------------- +# Pool shares and the zero rules +# --------------------------------------------------------------------------- + + +def test_the_correction_is_the_blocks_scaled_share_of_the_pool_total(): + share = PoolShare( + "share", "key", block_key_sum=2.0, pool_key_sum=12.0, weight_scale=3.0 + ) + assert share.factor() == pytest.approx(0.5) + assert share.as_receipt() == { + "formula": "share", + "key_variable": "key", + "correction": share.factor(), + } + + +def test_block_corrections_split_the_pool_by_their_key_totals(): + shares, scales = _block_shares(scaled=True) + total = sum(shares[block]["share"].factor() / scales[block] for block in BLOCKS) + assert total == pytest.approx(1.0, rel=1e-15) + assert shares["0"]["share"].pool_key_sum == pytest.approx(_frame_sum(np.arange(N))) + + +@pytest.mark.parametrize("scaled", [True, False]) +def test_corrected_blocks_give_the_pools_share_at_every_period(scaled): + pool = _pool() + shares, _scales = _block_shares(scaled=scaled) + for block, rows in BLOCKS.items(): + simulation = _block(block, scaled=scaled) + injected = inject_pool_shares( + simulation, shares[block], (D, P), weight_variable="weight" + ) + assert injected.periods == (str(D), str(P)) + for period in (D, P): + np.testing.assert_allclose( + simulation.calculate("share", period), + pool.calculate("share", period)[rows], + rtol=1e-6, + ) + np.testing.assert_array_equal( + simulation.baseline.calculate("share", period), + simulation.calculate("share", period), + ) + for target in (simulation, simulation.baseline): + assert str(period) in target.get_holder("share").inputs + + +def test_identical_copies_inject_their_own_values_bitwise(): + rows = np.arange(3) + copies = {"0": rows, "1": rows} + pool_mass = 2 * float(WEIGHT[rows].sum()) + shares = pool_shares( + {block: {"share": _frame_sum(rows)} for block in copies}, + {block: pool_mass / float(WEIGHT[rows].sum()) for block in copies}, + key_variables={"share": "key"}, + ) + for block in copies: + assert shares[block]["share"].factor() == 1.0 + simulation = FakeSimulation( + _fake_system(), _inputs(rows, weight_scale=2.0), rows=3 + ) + own = simulation.clone().calculate("share", P) + inject_pool_shares(simulation, shares[block], (P,), weight_variable="weight") + np.testing.assert_array_equal(simulation.calculate("share", P), own) + + +def test_a_block_without_keys_takes_zero(): + rows = np.arange(3) + inputs = _inputs(rows) + for period in (D, P): + inputs[("key_a", period)] = np.zeros(3) + inputs[("key_b", period)] = np.zeros(3) + simulation = FakeSimulation(_fake_system(), inputs, rows=3) + share = PoolShare("share", "key", block_key_sum=0.0, pool_key_sum=50.0) + assert share.factor() == 0.0 + assert check_key_sums( + simulation, {"share": share}, D, weight_variable="weight" + ) == {"share": 0.0} + inject_pool_shares(simulation, {"share": share}, (D, P), weight_variable="weight") + np.testing.assert_array_equal(simulation.calculate("share", P), np.zeros(3)) + + +def test_a_zero_block_total_over_nonzero_keys_is_refused(): + simulation = _block("0") + share = PoolShare("share", "key", block_key_sum=0.0, pool_key_sum=50.0) + with pytest.raises(PoolShareError, match="not every key"): + check_key_sums(simulation, {"share": share}, D, weight_variable="weight") + with pytest.raises(PoolShareError, match="not every key"): + inject_pool_shares(simulation, {"share": share}, (D,), weight_variable="weight") + + +def test_a_zero_pool_total_under_a_nonzero_block_total_is_refused(): + share = PoolShare("share", "key", block_key_sum=3.0, pool_key_sum=0.0) + with pytest.raises(PoolShareError, match="fallback branch"): + share.factor() + + +@pytest.mark.parametrize( + "fields", + [ + {"block_key_sum": math.nan}, + {"pool_key_sum": math.inf}, + {"weight_scale": 0.0}, + {"weight_scale": -2.0}, + ], +) +def test_a_correction_needs_finite_totals_and_a_positive_scale(fields): + values = {"block_key_sum": 1.0, "pool_key_sum": 2.0, "weight_scale": 1.0, **fields} + with pytest.raises(PoolShareError): + PoolShare("share", "key", **values) + + +def test_pool_shares_need_every_formula_in_every_block(): + with pytest.raises(PoolShareError, match="carries key sums"): + pool_shares({"0": {"share": 1.0}, "1": {}}, key_variables={"share": "key"}) + with pytest.raises(PoolShareError, match="weight factor"): + pool_shares({"0": {"share": 1.0}}, {"1": 1.0}, key_variables={"share": "key"}) + + +# --------------------------------------------------------------------------- +# Checks (a) and (b) +# --------------------------------------------------------------------------- + + +def test_key_sums_check_the_engines_total_against_the_frames(): + shares, _scales = _block_shares(scaled=True) + simulation = _block("1") + gaps = check_key_sums(simulation, shares["1"], D, weight_variable="weight") + assert gaps["share"] < 1e-6 + wrong = PoolShare( + "share", + "key", + block_key_sum=shares["1"]["share"].block_key_sum * 1.01, + pool_key_sum=shares["1"]["share"].pool_key_sum, + weight_scale=shares["1"]["share"].weight_scale, + ) + with pytest.raises(PoolShareError, match="frame total"): + check_key_sums(simulation, {"share": wrong}, D, weight_variable="weight") + + +def test_uniform_growth_returns_one_factor_per_name(): + simulation = _block("0", scaled=False) + factors = check_uniform_growth(simulation, ("key", "weight"), D, P) + assert factors["key"] == pytest.approx(GROWTH, rel=1e-6) + assert factors["weight"] == pytest.approx(1.0) + + +def test_growth_that_varies_by_row_is_refused(): + inputs = _inputs(BLOCKS["0"]) + # The third row's key grows by 1.0 where the others grow by 1.03. + inputs[("key_a", P)] = inputs[("key_a", D)] * np.array([1.03, 1.03, 1.0]) + simulation = FakeSimulation(_fake_system(), inputs, rows=3) + with pytest.raises(PoolShareError, match="one factor per row"): + check_uniform_growth(simulation, ("key",), D, P) + + +def test_the_growth_factor_is_the_ratio_of_totals(): + # Stored in float32, rows grown by one factor round apart: the factor is + # the totals' ratio, not any one row's. + inputs = _inputs(BLOCKS["0"]) + inputs[("key_a", D)] = np.array([10.1, 0.0, 30.7]) + inputs[("key_a", P)] = inputs[("key_a", D)] * GROWTH + simulation = FakeSimulation(_fake_system(), inputs, rows=3) + then = simulation.calculate("key", D).astype(np.float64) + now = simulation.calculate("key", P).astype(np.float64) + ratios = now / then + assert ratios.max() > ratios.min() + factor = check_uniform_growth(simulation, ("key",), D, P)["key"] + assert factor == pytest.approx(float(now.sum() / then.sum()), rel=1e-15) + assert all(factor != pytest.approx(ratio, rel=1e-12) for ratio in ratios) + + +def test_a_row_a_few_parts_in_a_hundred_thousand_off_is_refused(): + # Every row ends 1.2e-5 or more from the totals' factor, above the 1e-5 + # tolerance. + inputs = _inputs(BLOCKS["0"]) + inputs[("key_a", P)] = inputs[("key_a", D)] * GROWTH * np.array([1, 1, 1 + 3e-5]) + simulation = FakeSimulation(_fake_system(), inputs, rows=3) + with pytest.raises(PoolShareError, match="one factor per row"): + check_uniform_growth(simulation, ("key",), D, P) + + +def test_a_row_that_appears_after_the_dataset_year_is_refused(): + inputs = _inputs(BLOCKS["0"]) + # key_a is zero on the second row at D; give it a value at P only. + inputs[("key_a", P)] = np.array([10.3, 4.0, 30.9]) + inputs[("key_b", D)] = np.zeros(3) + inputs[("key_b", P)] = np.zeros(3) + simulation = FakeSimulation(_fake_system(), inputs, rows=3) + with pytest.raises(PoolShareError, match="zero at"): + check_uniform_growth(simulation, ("key",), D, P) + + +def test_a_name_no_row_carries_has_no_factor(): + inputs = _inputs(BLOCKS["0"]) + for period in (D, P): + inputs[("key_a", period)] = np.zeros(3) + inputs[("key_b", period)] = np.zeros(3) + simulation = FakeSimulation(_fake_system(), inputs, rows=3) + assert check_uniform_growth(simulation, ("key",), D, P) == {"key": None} + + +def test_factor_spread_is_relative_and_skips_blocks_without_carriers(): + spread = factor_spread( + {"0": {"key": 1.03, "weight": 1.0}, "1": {"key": 1.0300103, "weight": None}} + ) + assert spread["key"] == pytest.approx(1e-5, rel=1e-3) + assert spread["weight"] == 0.0 + + +# --------------------------------------------------------------------------- +# Injection into every simulation a block builds +# --------------------------------------------------------------------------- + + +def test_a_formula_already_an_input_is_not_injected_twice(): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0") + inject_pool_shares(simulation, shares["0"], (D,), weight_variable="weight") + with pytest.raises(PoolShareError, match="already an input"): + inject_pool_shares(simulation, shares["0"], (D,), weight_variable="weight") + + +def test_siblings_take_the_same_arrays_only_while_they_hold_the_same_values(): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0") + injected = inject_pool_shares( + simulation, shares["0"], (D, P), weight_variable="weight" + ) + sibling = _block("0") + injected.apply(sibling) + for period in (D, P): + np.testing.assert_array_equal( + sibling.calculate("share", period), simulation.calculate("share", period) + ) + np.testing.assert_array_equal( + sibling.baseline.calculate("share", period), + simulation.calculate("share", period), + ) + assert injected.matches(simulation) and injected.matches(sibling) + assert injected.as_receipt() == { + "periods": [str(D), str(P)], + "corrections": {"share": shares["0"]["share"].factor()}, + } + + substituted = _block("0") + substituted.set_input("key_a", P, np.zeros(3)) + with pytest.raises(PoolShareError, match="differs"): + injected.apply(substituted) + + sibling.set_input("weight", D, np.ones(3)) + assert not injected.matches(sibling) + + +def test_injection_reads_every_own_value_before_setting_any(): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0") + expected = { + period: simulation.clone().calculate("share", period) for period in (D, P) + } + injected = inject_pool_shares( + simulation, shares["0"], (P, D), weight_variable="weight" + ) + factor = injected.factors["share"] + for period in (D, P): + np.testing.assert_array_equal( + simulation.calculate("share", period), + (expected[period].astype(np.float64) * factor).astype(np.float32), + ) + + +def test_a_baseline_whose_own_shares_differ_is_refused(): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0") + simulation.baseline.set_input("key_a", D, KEY_A[BLOCKS["0"]] + 1.0) + with pytest.raises(PoolShareError, match="its baseline"): + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + + +def test_a_zero_block_whose_keys_turn_nonzero_later_is_refused(): + inputs = _inputs(BLOCKS["0"]) + inputs[("key_a", D)] = np.zeros(3) + inputs[("key_b", D)] = np.zeros(3) + simulation = FakeSimulation(_fake_system(), inputs, rows=3) + share = PoolShare("share", "key", block_key_sum=0.0, pool_key_sum=50.0) + with pytest.raises(PoolShareError, match=f"not every key is at {P}"): + inject_pool_shares( + simulation, {"share": share}, (D, P), weight_variable="weight" + ) + + +def test_a_sibling_whose_baseline_already_holds_the_shares_is_refused(): + shares, _scales = _block_shares(scaled=True) + injected = inject_pool_shares( + _block("0"), shares["0"], (D, P), weight_variable="weight" + ) + sibling = _block("0") + # The baseline's input is its own value, so only the refusal catches it. + own = sibling.baseline.clone().calculate("share", P) + sibling.baseline.set_input("share", P, own) + with pytest.raises(PoolShareError, match="already an input"): + injected.apply(sibling) + + +def test_a_sibling_baseline_changed_after_apply_no_longer_matches(): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0") + injected = inject_pool_shares( + simulation, shares["0"], (D, P), weight_variable="weight" + ) + sibling = _block("0") + injected.apply(sibling) + assert injected.matches(sibling) + sibling.baseline.set_input("weight", D, np.ones(3)) + assert not injected.matches(sibling) + + +def test_shares_go_in_before_branches_and_before_arming(fake_engine): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0") + simulation.get_branch("early") + with pytest.raises(PoolShareError, match="live branches"): + inject_pool_shares(simulation, shares["0"], (D,), weight_variable="weight") + simulation = _block("0") + simulation.baseline.get_branch("early") + with pytest.raises(PoolShareError, match="live branches"): + inject_pool_shares(simulation, shares["0"], (D,), weight_variable="weight") + + simulation = _block("0") + recorder = _recorder() + recorder.arm(simulation) + with pytest.raises(PoolShareError, match="before arming"): + inject_pool_shares(simulation, shares["0"], (D,), weight_variable="weight") + + injected = inject_pool_shares( + _block("0"), shares["0"], (D,), weight_variable="weight" + ) + sibling = _block("0") + recorder = _recorder() + recorder.arm(sibling) + with pytest.raises(PoolShareError, match="before arming"): + injected.apply(sibling) + sibling = _block("0") + sibling.get_branch("early") + with pytest.raises(PoolShareError, match="live branches"): + injected.apply(sibling) + + +# --------------------------------------------------------------------------- +# The recorder +# --------------------------------------------------------------------------- + + +def _recorder(declaration=None): + return FormulaRunRecorder.for_declaration(declaration or _declaration()) + + +def test_the_recorder_binds_per_instance_and_follows_core_clones(fake_engine): + simulation = _block("0") + recorder = _recorder() + method = vars(FakeSimulation)["_run_formula"] + recorder.arm(simulation) + assert "_run_formula" in vars(simulation) + assert "_run_formula" in vars(simulation.baseline) + assert vars(FakeSimulation)["_run_formula"] is method + assert recorder.is_armed(simulation) and recorder.is_armed(simulation.baseline) + + simulation.calculate("rates", P) + branch = simulation.get_branch("after_arming") + assert branch._run_formula.__self__ is branch + branch.calculate("local", D) + tokens = {run.simulation for run in recorder.runs} + assert recorder.token(branch) is not None + assert recorder.token(branch) != recorder.token(simulation) + assert recorder.token(simulation) in tokens + names = {(run.name, run.period) for run in recorder.runs} + assert ("share", str(P)) in names and ("rates", str(P)) in names + assert "local" in recorder.periods_run()[str(D)] + + recorder.disarm() + assert "_run_formula" not in vars(simulation) + assert "_run_formula" not in vars(branch) + before = len(recorder.runs) + simulation.calculate("rates", D) + assert len(recorder.runs) == before + + +def test_a_copy_that_inherits_a_token_gets_its_own(): + simulation = _block("0") + recorder = _recorder() + recorder.arm(simulation) + simulation.calculate("share", P) + copied = simulation.clone() + assert vars(copied)["_microcosm_formula_run_token"] == recorder.token(simulation) + assert recorder.token(copied) is None + copied.holders["share"].arrays.clear() + copied.calculate("share", P) + assert recorder.token(copied) not in (None, recorder.token(simulation)) + + +def test_predicates_and_zeros_are_read_at_the_run_in_that_simulation(): + simulation = _block("0") + recorder = _recorder() + recorder.arm(simulation) + simulation.calculate("budget_change", D) + branch = simulation.get_branch("reformed") + branch.tax_benefit_system = _fake_system(budget=5.0) + branch.calculate("budget_change", P) + runs = {run.period: run for run in recorder.runs if run.name == "budget_change"} + assert runs[str(D)].predicate_holds and runs[str(D)].all_zero + assert runs[str(P)].predicate_holds is False + assert runs[str(P)].all_zero is False + + +def test_arming_refuses_a_foreign_run_formula_but_replaces_a_disarmed_recorders(): + simulation = _block("0") + simulation._run_formula = types.MethodType( + lambda self, variable, population, period: None, simulation + ) + with pytest.raises(ValueError, match="own _run_formula"): + _recorder().arm(simulation) + + simulation = _block("0") + first = _recorder() + first.arm(simulation) + with pytest.raises(ValueError, match="own _run_formula"): + _recorder().arm(simulation) + copied = simulation.clone() + first.disarm() + second = _recorder() + second.arm(copied) + assert second.is_armed(copied) + + +def test_a_refused_arming_binds_nothing(): + simulation = _block("0") + simulation.baseline._run_formula = types.MethodType( + lambda self, variable, population, period: None, simulation.baseline + ) + recorder = _recorder() + with pytest.raises(ValueError, match="own _run_formula"): + recorder.arm(simulation) + assert "_run_formula" not in vars(simulation) + assert not recorder.is_armed(simulation) + simulation.calculate("rates", P) + assert recorder.runs == () and recorder.periods_run() == {} + + +class FakePeriod: + """A policyengine-core-style period: a unit, a start and a stop.""" + + def __init__(self, text, unit, first, last): + self.text, self.unit = text, unit + self.start = SimpleNamespace(year=first) + self.stop = SimpleNamespace(year=last) + + def __str__(self): + return self.text + + +@pytest.mark.parametrize( + "period, reaches", + [ + (2025, False), + (2026, True), + ("2025", False), + ("2025-12", False), + ("2026-01-01", True), + ("ETERNITY", True), + ("year:2024:2", True), # A form not read: taken to reach the window. + (FakePeriod("year:2025:2", "year", 2025, 2026), True), + (FakePeriod("year:2024:2", "year", 2024, 2025), False), + (FakePeriod("2025-12", "month", 2025, 2025), False), + (FakePeriod("ETERNITY", "eternity", 1, math.inf), True), + ], +) +def test_a_period_reaches_the_window_by_its_last_year(period, reaches): + window = PositionWindow(2026, f"{FAKE_ENGINE}:WINDOW", "positional draws") + assert window.contains(period) is reaches + + +def test_runs_reach_the_window_by_the_years_their_own_period_covers(fake_engine): + simulation, recorder = _armed_block() + across = FakePeriod("year:2025:2", "year", 2025, 2026) + month = FakePeriod("2025-12", "month", 2025, 2025) + simulation.calculate("local", across) + simulation.calculate("local", month) + assert recorder.period_span("year:2025:2") == (2025, 2026) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + window = {(v.name, v.period) for v in verdict.violations if v.kind == "window_run"} + assert ("local", "year:2025:2") in window + assert not any(period == "2025-12" for _name, period in window) + + +def test_predicates_must_be_for_watched_formulas(): + with pytest.raises(ValueError, match="unwatched"): + FormulaRunRecorder({"share"}, predicates={"other": ParameterTest("a", "eq", 0)}) + + +def test_the_walk_finds_values_calculated_before_arming(): + simulation = _block("0") + simulation.calculate("decile", P) + simulation.get_branch("subset").calculate("budget_change", D) + assert derived_watched_periods( + [simulation], ("decile", "budget_change", "share") + ) == { + "budget_change": (str(D),), + "decile": (str(P),), + } + + +# --------------------------------------------------------------------------- +# The judge +# --------------------------------------------------------------------------- + + +def _armed_block( + block="0", *, scaled=True, store=("months", "attendance"), system=None +): + shares, _scales = _block_shares(scaled=scaled) + simulation = _block(block, scaled=scaled, store=store, system=system) + inject_pool_shares(simulation, shares[block], (D, P), weight_variable="weight") + recorder = _recorder() + recorder.arm(simulation) + return simulation, recorder + + +def _kinds(verdict): + return {(v.kind, v.name, v.period) for v in verdict.violations} + + +def test_a_corrected_block_with_stored_inputs_and_zero_formulas_is_clean(fake_engine): + simulation, recorder = _armed_block() + for name in ("tax_total", "budget_change", "local"): + simulation.calculate(name, P) + verdict = judge( + _declaration(), MEASURE_PASS, recorder, [simulation], (D, P), [("tax_total", P)] + ) + assert verdict.clean, verdict.describe() + assert verdict.classes == { + "corrected": ("share",), + "stored": ("attendance", "months"), + "zero_at_policy": ("budget_change",), + "inert": (), + } + receipt = verdict.as_receipt() + assert receipt["pass"] == "measure" and receipt["clean"] is True + assert receipt["window"] == {"first_year": 2026, "violations": []} + assert receipt["refused"] == [] and receipt["violations"] == [] + assert receipt["declaration_sha256"] == _declaration().sha256 + assert receipt["engine"] == {"package": "fake-population-engine", "version": None} + + +def test_a_share_read_at_a_period_never_injected_is_a_violation(fake_engine): + simulation, recorder = _armed_block() + simulation.calculate("rates", 2023) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("corrected_derived", "share", "2023") in _kinds(verdict) + assert not verdict.clean + + +def test_a_share_not_injected_at_a_computed_period_is_a_violation(fake_engine): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0") + inject_pool_shares(simulation, shares["0"], (D,), weight_variable="weight") + recorder = _recorder() + recorder.arm(simulation) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("corrected_not_input", "share", str(P)) in _kinds(verdict) + + +def test_a_stored_formula_the_data_lacks_is_a_violation_in_the_measure_pass( + fake_engine, +): + simulation, recorder = _armed_block(store=("attendance",)) + simulation.calculate("months", P) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + assert { + ("stored_not_input", "months", str(D)), + ("stored_not_input", "months", str(P)), + ("stored_derived", "months", str(P)), + } <= _kinds(verdict) + + +def test_a_zero_formula_out_of_policy_or_nonzero_is_a_violation(fake_engine): + system = _fake_system(budget=5.0) + simulation, recorder = _armed_block(system=system) + simulation.calculate("budget_change", P) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("zero_at_policy", "budget_change", str(P)) in _kinds(verdict) + assert "budget_change" not in verdict.classes["zero_at_policy"] + + +def test_a_zero_formula_calculated_before_arming_is_judged_from_its_held_value( + fake_engine, +): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0", system=_fake_system(budget=5.0)) + simulation.calculate("budget_change", D) + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + recorder = _recorder() + recorder.arm(simulation) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("zero_at_policy", "budget_change", str(D)) in _kinds(verdict) + + +def _reformed_branch(simulation, name, **system): + branch = simulation.get_branch(name) + branch.tax_benefit_system = _fake_system(**system) + return branch + + +def test_a_held_zero_formula_is_judged_where_it_is_held(fake_engine): + # A branch made before arming holds a nonzero value; the main simulation + # then runs the formula cleanly at the same period. The clean run says + # nothing of the branch's value. + shares, _scales = _block_shares(scaled=True) + simulation = _block("0") + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + _reformed_branch(simulation, "reformed", budget=5.0).calculate("budget_change", P) + recorder = _recorder() + recorder.arm(simulation) + simulation.calculate("budget_change", P) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("zero_at_policy", "budget_change", str(P)) in _kinds(verdict) + assert "budget_change" not in verdict.classes["zero_at_policy"] + + +@pytest.mark.parametrize( + "predicate, budget", + [ + # The predicate holds but the value is not zero. + (ParameterTest("gov.vat_rate", "gt", 0), 5.0), + # The value is zero but the predicate fails. + (ParameterTest("gov.vat_rate", "eq", 0), 0.0), + ], +) +def test_a_zero_formula_needs_its_predicate_and_zero_values( + fake_engine, predicate, budget +): + declaration = _declaration(zero_at_policy={"budget_change": predicate}) + shares, _scales = _block_shares(scaled=True) + simulation = _block("0", system=_fake_system(budget=budget)) + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + recorder = FormulaRunRecorder.for_declaration(declaration) + recorder.arm(simulation) + simulation.calculate("budget_change", P) + verdict = judge(declaration, MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("zero_at_policy", "budget_change", str(P)) in _kinds(verdict) + + # The same, held from before arming, is judged from the held value. + simulation = _block("0", system=_fake_system(budget=budget)) + simulation.calculate("budget_change", D) + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + recorder = FormulaRunRecorder.for_declaration(declaration) + recorder.arm(simulation) + verdict = judge(declaration, MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("zero_at_policy", "budget_change", str(D)) in _kinds(verdict) + + +def test_values_held_from_before_arming_are_judged_as_runs(fake_engine): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0", store=()) + simulation.calculate("share", 2023) + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + for name in ("decile", "months", "tax_total"): + simulation.calculate(name, P) + recorder = _recorder() + recorder.arm(simulation) + assert recorder.runs == () + with pytest.raises(PopulationGuardError) as raised: + judge(_declaration(), SPINE_BATCHED_PASS, recorder, [simulation], (D, P)) + assert { + ("corrected_derived", "share", "2023"), + ("refused_ran", "decile", str(P)), + ("stored_derived", "months", str(P)), + ("inert_carrier", "tax_total", str(P)), + } <= _kinds(raised.value.verdict) + + +def test_a_baseline_without_the_injected_shares_is_a_violation(fake_engine): + simulation, recorder = _armed_block() + holder = simulation.baseline.get_holder("share") + holder.inputs.discard(str(P)) + holder.arrays.pop(str(P)) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("corrected_not_input", "share", str(P)) in _kinds(verdict) + + +def test_attendance_may_run_in_a_spine_batch_through_a_zero_sink(fake_engine): + simulation, recorder = _armed_block(store=("months",)) + simulation.calculate("vat_change", P) + verdict = judge(_declaration(), SPINE_BATCHED_PASS, recorder, [simulation], (D, P)) + assert verdict.clean, verdict.describe() + assert verdict.classes["inert"] == ("attendance",) + + raising = PassPolicy("measure_raise", inert_allowed=False, on_violation="raise") + with pytest.raises(PopulationGuardError) as raised: + judge(_declaration(), raising, recorder, [simulation], (D, P)) + assert ("stored_not_input", "attendance", str(D)) in _kinds(raised.value.verdict) + assert "stored_not_input attendance@2024" in str(raised.value) + + +def test_attendance_that_reaches_a_level_or_a_nonzero_sink_raises_in_the_spine( + fake_engine, +): + simulation, recorder = _armed_block(store=("months",)) + simulation.calculate("tax_total", P) + with pytest.raises(PopulationGuardError) as raised: + judge(_declaration(), SPINE_BATCHED_PASS, recorder, [simulation], (D, P)) + assert { + ("inert_carrier", "tax_total", str(P)), + ("inert_without_sink", "attendance", str(P)), + } <= _kinds(raised.value.verdict) + + reformed = _fake_system(vat_rate=0.25) + simulation, recorder = _armed_block(store=("months",), system=reformed) + simulation.calculate("vat_change", P) + with pytest.raises(PopulationGuardError) as raised: + judge(_declaration(), SPINE_BATCHED_PASS, recorder, [simulation], (D, P)) + assert { + ("inert_sink", "vat_change", str(P)), + ("inert_predicate", "attendance", str(P)), + } <= _kinds(raised.value.verdict) + + simulation, recorder = _armed_block(store=("months",)) + with pytest.raises(PopulationGuardError) as raised: + judge( + _declaration(), + SPINE_BATCHED_PASS, + recorder, + [simulation], + (D, P), + [("vat", P)], + ) + assert ("inert_request", "vat", str(P)) in _kinds(raised.value.verdict) + + +def test_a_held_sink_is_judged_where_it_is_held(fake_engine): + # A branch made before arming, under a reformed rate, holds a nonzero + # sink; the main simulation's clean run of the sink at the same period + # says nothing of it. + shares, _scales = _block_shares(scaled=True) + simulation = _block("0", store=("months",)) + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + reformed = _reformed_branch(simulation, "reformed", vat_rate=0.25) + assert np.any(reformed.calculate("vat_change", P) != 0) + recorder = _recorder() + recorder.arm(simulation) + assert np.all(simulation.calculate("vat_change", P) == 0) + with pytest.raises(PopulationGuardError) as raised: + judge(_declaration(), SPINE_BATCHED_PASS, recorder, [simulation], (D, P)) + assert ("inert_sink", "vat_change", str(P)) in _kinds(raised.value.verdict) + + +def test_every_held_sink_at_a_period_is_judged(fake_engine): + shares, _scales = _block_shares(scaled=True) + simulation = _block("0", store=("months",)) + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + simulation.calculate("vat_change", P) + _reformed_branch(simulation, "reformed", vat_rate=0.25).calculate("vat_change", P) + recorder = _recorder() + recorder.arm(simulation) + with pytest.raises(PopulationGuardError) as raised: + judge(_declaration(), SPINE_BATCHED_PASS, recorder, [simulation], (D, P)) + assert ("inert_sink", "vat_change", str(P)) in _kinds(raised.value.verdict) + + +def _inert_declaration(predicate): + rule = InertRule( + sink="vat_change", + levels=("vat", "baseline_vat"), + carriers=("tax_total",), + predicate=predicate, + ) + return _declaration(inert={"attendance": rule}) + + +@pytest.mark.parametrize( + "predicate, vat_rate", + [ + # The predicate holds but the sink is not zero. + (ParameterTest("gov.budget", "eq", 0), 0.25), + # The sink is zero but the predicate fails. + (ParameterTest("gov.budget", "ne", 0), 0.2), + ], +) +def test_a_sink_needs_its_predicate_and_zero_values(fake_engine, predicate, vat_rate): + declaration = _inert_declaration(predicate) + shares, _scales = _block_shares(scaled=True) + simulation = _block("0", store=("months",), system=_fake_system(vat_rate=vat_rate)) + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + recorder = FormulaRunRecorder.for_declaration(declaration) + recorder.arm(simulation) + simulation.calculate("vat_change", P) + with pytest.raises(PopulationGuardError) as raised: + judge(declaration, SPINE_BATCHED_PASS, recorder, [simulation], (D, P)) + assert ("inert_sink", "vat_change", str(P)) in _kinds(raised.value.verdict) + + # The same, held from before arming, is judged from the held value. + simulation = _block("0", store=("months",), system=_fake_system(vat_rate=vat_rate)) + inject_pool_shares(simulation, shares["0"], (D, P), weight_variable="weight") + simulation.calculate("vat_change", P) + recorder = FormulaRunRecorder.for_declaration(declaration) + recorder.arm(simulation) + with pytest.raises(PopulationGuardError) as raised: + judge(declaration, SPINE_BATCHED_PASS, recorder, [simulation], (D, P)) + assert ("inert_sink", "vat_change", str(P)) in _kinds(raised.value.verdict) + + +def test_a_refused_rank_is_a_violation_run_or_requested(fake_engine): + simulation, recorder = _armed_block() + simulation.calculate("decile", P) + verdict = judge( + _declaration(), MEASURE_PASS, recorder, [simulation], (D, P), [("decile", P)] + ) + assert { + ("refused_ran", "decile", str(P)), + ("refused_request", "decile", str(P)), + } <= _kinds(verdict) + assert verdict.as_receipt()["refused"] == ["decile"] + + +def test_work_in_the_position_window_is_a_violation(fake_engine): + simulation, recorder = _armed_block() + simulation.calculate("local", 2026) + verdict = judge( + _declaration(), + MEASURE_PASS, + recorder, + [simulation], + (D, P, 2027), + [("local", 2028)], + ) + expected = { + ("window_run", "local", "2026"), + ("window_run", "key_a", "2026"), + ("window_period", None, "2027"), + ("window_request", "local", "2028"), + } + assert expected <= _kinds(verdict) + window = verdict.as_receipt()["window"] + assert window["first_year"] == 2026 + assert { + (item["kind"], item["name"], item["period"]) for item in window["violations"] + } == expected + + +def test_a_structural_reform_switched_on_is_a_violation(fake_engine): + simulation, recorder = _armed_block() + declaration = _declaration( + constructor_factories=( + f"{FAKE_ENGINE}:quiet_reform", + f"{FAKE_ENGINE}:loud_reform", + ) + ) + verdict = judge(declaration, MEASURE_PASS, recorder, [simulation], (D, P)) + assert ("constructor_reform", f"{FAKE_ENGINE}:loud_reform", str(D)) in _kinds( + verdict + ) + + +def test_a_structural_reform_switched_on_at_a_later_period_is_a_violation( + fake_engine, +): + simulation, recorder = _armed_block() + declaration = _declaration(constructor_factories=(f"{FAKE_ENGINE}:late_reform",)) + verdict = judge(declaration, MEASURE_PASS, recorder, [simulation], (D, P)) + assert _kinds(verdict) == { + ("constructor_reform", f"{FAKE_ENGINE}:late_reform", str(P)) + } + + +def test_a_verdict_lists_at_most_fifty_violations(fake_engine): + simulation, recorder = _armed_block() + requested = [("decile", year) for year in range(1950, 2010)] + verdict = judge( + _declaration(), MEASURE_PASS, recorder, [simulation], (D, P), requested + ) + assert verdict.truncated and len(verdict.violations) == 50 + assert verdict.as_receipt()["truncated"] is True + assert verdict.describe().endswith("(and more).") + + +def test_a_simulation_the_recorder_never_armed_is_a_violation(fake_engine): + simulation, recorder = _armed_block() + sibling = _block("0") + verdict = judge( + _declaration(), MEASURE_PASS, recorder, [simulation, sibling], (D, P) + ) + assert ("not_armed", None, None) in _kinds(verdict) + + +def test_a_verdict_carries_names_periods_and_booleans_only(fake_engine): + simulation, recorder = _armed_block(store=()) + for period in (2023, 2026): + simulation.calculate("tax_total", period) + simulation.calculate("decile", period) + verdict = judge(_declaration(), MEASURE_PASS, recorder, [simulation], (D, P)) + + def leaves(value): + if isinstance(value, dict): + for item in value.values(): + yield from leaves(item) + elif isinstance(value, list): + for item in value: + yield from leaves(item) + else: + yield value + + for leaf in leaves(verdict.as_receipt()): + assert leaf is None or isinstance(leaf, (str, bool, int)), leaf + assert not isinstance(leaf, float) + + +# --------------------------------------------------------------------------- +# The declaration +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "changes, message", + [ + ({"refused": ("decile", "share")}, "both"), + ({"stored": ("months",)}, "inert formula must also be declared stored"), + ({"corrected": {"share": "other_key"}}, "name no inputs"), + ({"refused": ("decile", "decile")}, "repeat"), + ], +) +def test_a_declaration_refuses_overlapping_or_incomplete_classes(changes, message): + with pytest.raises(ValueError, match=message): + _declaration(**changes) + + +def test_the_declaration_digest_follows_its_content(): + assert _declaration().sha256 == _declaration().sha256 + assert _declaration().sha256 != _declaration(observed=()).sha256 + assert _declaration().classified() == { + "share", + "months", + "attendance", + "budget_change", + "decile", + } + assert _declaration().watched() == _declaration().classified() | { + "rates", + "vat", + "baseline_vat", + "vat_change", + "tax_total", + } + assert set(_declaration().run_predicates()) == { + "budget_change", + "attendance", + "vat_change", + } + + +def test_predicates_combine_parameter_tests(): + parameters = _fake_system(budget=0.0).parameters(D) + assert ParameterTest("gov.budget", "eq", 0).holds(parameters) + assert ParameterTest("gov.vat_rate", "gt", 0.1).holds(parameters) + assert any_of( + ParameterTest("gov.budget", "ne", 0), ParameterTest("gov.vat_rate", "le", 0.2) + ).holds(parameters) + assert not all_of( + ParameterTest("gov.budget", "eq", 0), ParameterTest("gov.vat_rate", "lt", 0.2) + ).holds(parameters) + with pytest.raises(ValueError): + ParameterTest("gov.budget", "approx", 0) + with pytest.raises(ValueError): + ParameterTest("gov.budget", "eq_path", 0) + with pytest.raises(ValueError): + all_of() + + +@pytest.fixture +def pinned_package(tmp_path, monkeypatch): + package = tmp_path / "fake_pinned_engine" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "share.py").write_text( + "def share_formula(key, total):\n return key / total\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + import importlib + + importlib.invalidate_caches() + yield package + sys.modules.pop("fake_pinned_engine.share", None) + sys.modules.pop("fake_pinned_engine", None) + + +def test_source_pins_hash_files_and_objects(pinned_package): + import hashlib + + path = pinned_package / "share.py" + assert source_sha256("fake_pinned_engine/share.py") == ( + hashlib.sha256(path.read_bytes()).hexdigest() + ) + assert source_sha256("fake_pinned_engine.share:share_formula") == ( + hashlib.sha256(path.read_text().encode()).hexdigest() + ) + with pytest.raises(FileNotFoundError): + source_sha256("fake_pinned_engine/missing.py") + + +def test_validation_passes_on_the_engine_it_describes(fake_engine, pinned_package): + digest = source_sha256("fake_pinned_engine/share.py") + declaration = _declaration(digests={"fake_pinned_engine/share.py": digest}) + validate_declaration(declaration, _fake_system()) + + +def test_validation_fails_closed_on_every_kind_of_drift(fake_engine, pinned_package): + digest = source_sha256("fake_pinned_engine/share.py") + declaration = _declaration( + digests={"fake_pinned_engine/share.py": digest}, + constructor_factories=(f"{FAKE_ENGINE}:WINDOW",), + zero_at_policy={"budget_change": ParameterTest("gov.missing", "eq", 0)}, + ) + fake_engine.WHOLE_POPULATION = fake_engine.WHOLE_POPULATION | {"extra"} + fake_engine.WINDOW = range(2027, 2033) + (pinned_package / "share.py").write_text("def share_formula():\n return 1\n") + system = _fake_system() + system.variables["key"].adds = ["key_a"] + with pytest.raises(PopulationDeclarationError) as raised: + validate_declaration(declaration, system) + message = str(raised.value) + for fragment in ( + "lists ['extra']", + "adds ['key_a']", + "fake_pinned_engine/share.py has changed", + "no longer starts the window at 2026", + "is not callable", + "no parameter gov.missing", + ): + assert fragment in message, message + + +def test_validation_requires_formulas_over_plain_inputs(fake_engine): + system = _fake_system() + system.variables["share"].formulas = {} + system.variables["key_b"].formulas = {"0000-01-01": _local} + with pytest.raises(PopulationDeclarationError) as raised: + validate_declaration(_declaration(), system) + assert "share is no longer a formula" in str(raised.value) + assert "key_b is no longer a plain household input" in str(raised.value) + + +@pytest.mark.parametrize( + "change, fragment", + [ + ( + lambda variables: setattr(variables["key"], "subtracts", ["key_b"]), + "key is no longer a plain sum of its inputs", + ), + ( + lambda variables: setattr( + variables["key"], "formulas", {"0000-01-01": _local} + ), + "key is no longer a plain sum of its inputs", + ), + ( + lambda variables: setattr( + variables["share"], "entity", SimpleNamespace(key="person") + ), + "share, key and weight no longer share an entity", + ), + ( + lambda variables: setattr( + variables["weight"], "entity", SimpleNamespace(key="person") + ), + "share, key and weight no longer share an entity", + ), + ( + lambda variables: setattr( + variables["key_a"], "entity", SimpleNamespace(key="person") + ), + "key_a is no longer a plain household input", + ), + ], +) +def test_validation_requires_one_entity_and_a_plain_key(fake_engine, change, fragment): + system = _fake_system() + change(system.variables) + with pytest.raises(PopulationDeclarationError) as raised: + validate_declaration(_declaration(), system) + assert fragment in str(raised.value) + + +def test_pass_policies_record_or_raise(): + assert MEASURE_PASS.on_violation == "record" and not MEASURE_PASS.inert_allowed + assert ( + SPINE_BATCHED_PASS.on_violation == "raise" and SPINE_BATCHED_PASS.inert_allowed + ) + with pytest.raises(ValueError): + PassPolicy("other", inert_allowed=False, on_violation="warn") + + +# --------------------------------------------------------------------------- +# The float32 bound +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "n, roundings", + [(1, 0), (7, 6), (8, 3), (15, 10), (127, 24), (128, 18), (129, 12), (255, 19)], +) +def test_sum_roundings_follow_numpys_pairwise_leaves(n, roundings): + assert float32_sum_roundings(n) == roundings + + +def test_the_leaf_term_does_not_shrink_with_n(): + assert float32_sum_roundings(2**20) - float32_sum_roundings(2**10) == 10 + assert float32_sum_roundings(1024) > math.log2(1024) + + +@pytest.mark.parametrize("n", [5, 100, 1_000, 40_000, 300_000]) +def test_numpys_float32_sums_stay_inside_the_bound(n): + rng = np.random.default_rng(n) + for values in ( + rng.lognormal(3.0, 2.0, n).astype(np.float32), + np.full(n, np.float32(1.0) + np.float32(2.0**-23)), + ): + exact = math.fsum(values.astype(np.float64)) + error = abs(float(values.sum(dtype=np.float32)) - exact) + assert error <= float32_sum_roundings(n) * FLOAT32_UNIT_ROUNDOFF * exact + + +def test_the_share_bound_grows_with_the_sums_and_the_projection(): + base = pool_share_rtol(8_000, 120_000, key_inputs=1) + assert base == pytest.approx( + (float32_sum_roundings(8_000) + float32_sum_roundings(120_000) + 13) + * FLOAT32_UNIT_ROUNDOFF + ) + assert pool_share_rtol( + 8_000, 120_000, key_inputs=1, projection_years=1 + ) == pytest.approx(base + 4 * FLOAT32_UNIT_ROUNDOFF) + assert pool_share_rtol(80_000, 1_200_000, key_inputs=12) < 1e-5 + + +def test_the_share_bound_counts_the_adds_that_form_the_key(): + # Each of a key's float32 adds can round in the block's sum one way and + # in the pool's the other: two roundings per add beyond the first input. + one = pool_share_rtol(1, 2, key_inputs=1) + assert pool_share_rtol(1, 2, key_inputs=2) == pytest.approx( + one + 2 * FLOAT32_UNIT_ROUNDOFF + ) + assert pool_share_rtol(1, 2, key_inputs=12) == pytest.approx( + one + 22 * FLOAT32_UNIT_ROUNDOFF + ) + with pytest.raises(ValueError): + pool_share_rtol(1, 2, key_inputs=0) + + +def _engine_adds(parts): + """A key as policyengine-core's adds form it: 0 plus each float32 input + in turn, in float32.""" + + total = 0 + for part in parts: + total = total + np.asarray(part, dtype=np.float32) + return np.asarray(total, dtype=np.float32) + + +def test_a_twelve_input_key_that_rounds_apart_stays_inside_the_bound(): + # Two made-up one-person households. The block's household's eleven small + # inputs each round down when added to its first, the rest of the pool's + # each round up; the frame sums them in float64, as c_b is formed. + u = 2.0**-24 + down = np.float32(u * (1 - 2.0**-23)) + up = np.float32(u * (1 + 2.0**-22)) + parts = { + "block": [np.float32(1.0)] + [down] * 11, + "rest": [np.float32(1.0)] + [up] * 11, + } + weight = {"block": 1.0, "rest": 1023.0} + key = {name: _engine_adds([[part] for part in parts[name]]) for name in parts} + frame = { + name: math.fsum(float(part) for part in parts[name]) * weight[name] + for name in parts + } + scale = (weight["block"] + weight["rest"]) / weight["block"] + + block_weights = np.array([weight["block"] * scale], dtype=np.float32) + block_value = key["block"] / (key["block"] * block_weights).sum() + pool_keys = np.concatenate([key["block"], key["rest"]]) + pool_weights = np.array([weight["block"], weight["rest"]], dtype=np.float32) + pool_value = (pool_keys / (pool_keys * pool_weights).sum())[0] + share = PoolShare( + "consumption_shareholding", + "consumption", + block_key_sum=frame["block"], + pool_key_sum=frame["block"] + frame["rest"], + weight_scale=scale, + ) + corrected = np.float32(float(block_value[0]) * share.factor()) + gap = abs(float(corrected) / float(pool_value) - 1) + assert gap > pool_share_rtol(1, 2, key_inputs=1) + assert gap <= pool_share_rtol(1, 2, key_inputs=12) + + +def test_the_per_row_bound_covers_a_value_netting_large_terms(): + # A value adding a share term to large household-local terms that + # nearly cancel, in float32, as a household's net income does: relative + # to itself its gap is far above the share bound, within the per-row one. + rng = np.random.default_rng(7) + rows = 20_000 + rtol = pool_share_rtol(5_000, 20_000, key_inputs=2) + income = rng.uniform(1e5, 1e6, rows).astype(np.float32) + taxes = (income * np.float32(0.999)).astype(np.float32) + pool_share = rng.uniform(0.0, 300.0, rows).astype(np.float32) + off = rng.uniform(-rtol, rtol, rows) * 0.5 + block_share = (pool_share.astype(np.float64) * (1 + off)).astype(np.float32) + + def net(share): + return (income - taxes - share).astype(np.float32) + + gap = np.abs(net(block_share).astype(np.float64) - net(pool_share)) + terms = [income, taxes, pool_share] + magnitude = sum(np.abs(term.astype(np.float64)) for term in terms) + atol = pool_share_atol(pool_share, magnitude, share_rtol=rtol, additions=2) + assert np.all(gap <= atol) + relative = gap / np.maximum(np.abs(net(pool_share)), 1e-30) + assert relative.max() > rtol + np.testing.assert_allclose( + pool_share_atol([2.0], [10.0], share_rtol=1e-6, additions=3), + [2e-6 + 6 * FLOAT32_UNIT_ROUNDOFF * 10.0], + ) + with pytest.raises(ValueError): + pool_share_atol([1.0], [1.0], share_rtol=1e-6, additions=-1) + + +# --------------------------------------------------------------------------- +# Household batches and the scatter back +# --------------------------------------------------------------------------- + + +def _clone_frame(): + """Three clones of two households; people listed out of id order.""" + + household = pd.DataFrame( + { + "household_id": np.array([11, 12, 21, 22, 31, 32], dtype=np.int64), + "household_clone_index": [2, 0, 1, 2, 0, 1], + } + ) + person = pd.DataFrame( + { + "person_id": np.array( + [322, 111, 211, 121, 221, 311, 112, 321], dtype=np.int64 + ), + "person_benunit_id": np.array( + [32, 11, 21, 12, 22, 31, 11, 32], dtype=np.int64 + ), + "person_household_id": np.array( + [32, 11, 21, 12, 22, 31, 11, 32], dtype=np.int64 + ), + } + ) + benunit = pd.DataFrame( + {"benunit_id": np.array([11, 12, 21, 22, 31, 32], dtype=np.int64)} + ) + return Frame( + {"person": person, "benunit": benunit, "household": household}, + EntitySchema(group_entities=("benunit", "household")), + { + "household": Weights( + np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), kind=WeightKind.DESIGN + ) + }, + ) + + +def test_household_batches_follow_the_column_in_ascending_order(): + frame = _clone_frame() + batches = household_batches(frame, "household_clone_index") + assert [batch.key for batch in batches] == [0, 1, 2] + household_ids = frame.table("household")["household_id"].to_numpy() + person_households = frame.table("person")["person_household_id"].to_numpy() + for batch in batches: + ids = household_ids[batch.households] + np.testing.assert_array_equal(batch.persons, np.isin(person_households, ids)) + assert sum(int(batch.persons.sum()) for batch in batches) == frame.n("person") + again = household_batches(frame, "household_clone_index") + for first, second in zip(batches, again, strict=True): + assert first.key == second.key + np.testing.assert_array_equal(first.persons, second.persons) + + +def test_batches_need_the_column_and_whole_households(): + frame = _clone_frame() + with pytest.raises(ValueError, match="carries no"): + household_batches(frame, "missing_column") + person = frame.table("person").copy() + person.loc[person["person_id"] == 112, "person_benunit_id"] = 12 + split = Frame( + { + "person": person, + "benunit": frame.table("benunit"), + "household": frame.table("household"), + }, + frame.schema, + {"household": frame.weights_for("household")}, + ) + with pytest.raises(ValueError, match="must nest in households"): + assert_groups_nest_in_households(split) + with pytest.raises(ValueError, match="must nest in households"): + household_batches(split, "household_clone_index") + + +@pytest.mark.parametrize("entity", ["person", "benunit", "household"]) +def test_scatter_puts_batch_results_in_one_engines_order(entity): + frame = _clone_frame() + column = frame.schema.entity_id_column(entity) + expected = frame.table(entity)[column].to_numpy() * 10 + 1 + parts = [] + for batch in reversed(household_batches(frame, "household_clone_index")): + ids = frame.select(batch.persons).table(entity)[column].to_numpy()[::-1] + parts.append((ids, ids * 10 + 1)) + np.testing.assert_array_equal(scatter_by_entity_ids(frame, entity, parts), expected) + + +def test_scatter_refuses_missing_repeated_or_mistyped_rows(): + frame = _clone_frame() + ids = frame.table("household")["household_id"].to_numpy() + values = np.arange(len(ids), dtype=np.float32) + with pytest.raises(ValueError, match="different number"): + scatter_by_entity_ids(frame, "household", [(ids[:-1], values[:-1])]) + with pytest.raises(ValueError, match="same household row"): + repeated = ids.copy() + repeated[-1] = repeated[0] + scatter_by_entity_ids(frame, "household", [(repeated, values)]) + with pytest.raises(ValueError, match="does not hold"): + unknown = ids.copy() + unknown[0] = 99 + scatter_by_entity_ids(frame, "household", [(unknown, values)]) + with pytest.raises(ValueError, match="different types"): + scatter_by_entity_ids( + frame, + "household", + [(ids[:3], values[:3]), (ids[3:], values[3:].astype(np.float64))], + ) From 6901fd909c0e38b5107cec7f88039b7d94e1447d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 06:14:28 +0000 Subject: [PATCH 002/108] Carry the other spells by the engine's growth parameters (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The annual-income simulation carries each input's other spells from the dataset's year by the factor the engine projects that input by. The factor was the median of the engine's own year-on-year ratios over the rows the simulation holds. That is a statistic of whichever block ran, so per-clone engine blocks that differ would carry the same spell by different factors, and no block would match one engine over the pool. Following María's ruling of 6 Nov, the factor is now the product of one plus the engine's own rate, for each year after the dataset's, in the growth series policyengine-uk's dataset projection uses for the input: average earnings for pay, mixed income per head for profit. The rates are read from the simulation's parameters. The engine's rows must still agree with the factor within 1e-5, or the resolver refuses, as before. This moves data on today's national and dense lines. The median sat within float32 rounding of the parameter, so an annual input with other spells can move by one float32 step. The *_annual matrix entries, calibrated weights and H5 bytes can move, and other_spells_engine_factors changes in every resolver receipt that carries it. A K=1 parity check should expect *_annual inputs within one float32 step, not an identical H5. The full target and problem kernels and the national target kernels re-key with measure_simulation; no spine stage does. An engine-free test holds the factor to the parameters and to the rows' tolerance. An engine test pins the declared series to policyengine-uk's own uprating map and checks the factor on a synthetic frame. Co-Authored-By: Claude Opus 5.5 --- .../build/uk_runtime/measure_simulation.py | 52 +++++++++++-- .../uk/test_uk_annual_income_measures.py | 57 ++++++++++++++ .../uk/test_uk_measure_simulation.py | 77 ++++++++++++++++++- 3 files changed, 178 insertions(+), 8 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py index ef2cff31f5..66542cadfe 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -421,6 +421,24 @@ def annuity_and_trust_share(frame: Any) -> np.ndarray: ("esa_contrib_reported", CONTRIBUTORY_BENEFITS_ANNUAL_SHARE, None), ("incapacity_benefit_reported", CONTRIBUTORY_BENEFITS_ANNUAL_SHARE, None), ) +#: The growth series policyengine-uk's dataset projection carries each input +#: with other spells by (its ``data/uprating_indices.yaml``: pay, stored as +#: ``employment_income``, by average earnings; profit by mixed income per +#: head). The other spells grow from the dataset's year by the product of the +#: engine's own rates in that series, read from its parameters, so the factor +#: is the same in every per-clone engine block and in one engine over the pool +#: (María's ruling of 2026-11-06, microcosm#931; it was the median of the +#: engine's own year-on-year ratios, a statistic of whichever block ran). +_ANNUAL_INCOME_OTHER_SPELLS_GROWTH: Mapping[str, str] = MappingProxyType( + { + "employment_income_before_lsr": ( + "gov.economic_assumptions.yoy_growth.obr.average_earnings" + ), + "self_employment_income": ( + "gov.economic_assumptions.yoy_growth.obr.per_capita.mixed_income" + ), + } +) #: Taxable benefits whose entitlement reads earnings, held at their baseline #: values: a carer was paid in the year at the earnings of the time, which the #: current concept records, not at the year's total. @@ -1772,27 +1790,47 @@ def _annual_income_simulation(self) -> Any: def _engine_projection_factor(self, name: str) -> float: """The factor the engine carries ``name`` by from the dataset's year. - Read off the engine's own projection, so the other spells grow by the - series the current amount grows by; a projection that is not one - factor for every row refuses, since no single factor then carries the - other spells as the engine carries pay. + The product, over each year after the dataset's up to this one, of + one plus the engine's own growth rate in the series its dataset + projection carries ``name`` by (``_ANNUAL_INCOME_OTHER_SPELLS_GROWTH``), + read from the simulation's parameters as the projection reads them. + A parameter, not a statistic of the rows, so every engine block and + one engine over the pool carry the other spells alike (microcosm#931). + The engine's own arrays must agree with it, within 1e-5, on every row + that carries ``name``; otherwise no single factor carries the other + spells as the engine carries pay, and it refuses. """ + series = _ANNUAL_INCOME_OTHER_SPELLS_GROWTH.get(name) + if series is None: + raise ValueError( + f"No engine growth series is declared for {name}, so its other " + "spells cannot be carried to the measurement year." + ) dataset_year = self._dataset_year() + if self.year < dataset_year: + raise ValueError( + f"The engine projects the dataset from {dataset_year} forward, so " + f"the other spells of {name} cannot be carried back to {self.year}." + ) + growth = self.simulation.tax_benefit_system.parameters.get_child(series) + factor = 1.0 + for year in range(dataset_year + 1, self.year + 1): + factor *= 1.0 + float(growth(str(year))) then = _values(self.simulation.calculate(name, dataset_year)).astype(float) now = _values(self.simulation.calculate(name, self.year)).astype(float) carried = np.abs(then) > 0 if not carried.any(): raise ValueError( f"No row carries {name} in {dataset_year}, so the engine's factor " - f"to {self.year} for the other spells cannot be read." + f"to {self.year} for the other spells cannot be checked." ) factors = now[carried] / then[carried] - factor = float(np.median(factors)) if (np.abs(factors - factor) > 1e-5 * max(abs(factor), 1.0)).any(): raise ValueError( f"The engine does not carry {name} from {dataset_year} to " - f"{self.year} by one factor, so the other spells cannot follow it." + f"{self.year} by one factor, its growth series {series}, so the " + "other spells cannot follow it." ) self._annual_income_factors[name] = factor return factor diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_annual_income_measures.py b/packages/microcosm-build/tests/engine/uk/test_uk_annual_income_measures.py index 0488b408ad..b465c89d1a 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_annual_income_measures.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_annual_income_measures.py @@ -167,6 +167,63 @@ def test_the_annual_measures_tax_each_persons_year(tmp_path) -> None: assert set(receipt["computed_measures"]) == set(UK_ANNUAL_INCOME_MEASURES) +def test_the_other_spells_grow_by_the_engines_own_growth_series(tmp_path) -> None: + """The other spells grow by the series policyengine-uk's dataset + projection carries the input by, read from the engine's parameters: the + same factor in every engine block and in one engine over the pool + (microcosm#931). Pinned to the engine's own uprating map, and to no path + that projects the input otherwise.""" + + from policyengine_uk.data import economic_assumptions + + from microcosm.build.uk_runtime import measure_simulation + + declared = measure_simulation._ANNUAL_INCOME_OTHER_SPELLS_GROWTH + with_other_spells = { + name + for name, _share, other in measure_simulation._ANNUAL_INCOME_INPUTS + if other is not None + } + assert set(declared) == with_other_spells + uprating = economic_assumptions._uprating_index_map() + # The dataset stores pay as employment_income, which the simulation moves + # to employment_income_before_lsr once projected. + stored_as = {"employment_income_before_lsr": "employment_income"} + custom = ( + set(economic_assumptions.DATASET_UPRATING_EXCLUSIONS) + | set(economic_assumptions.COUNTRY_UPRATED_INPUTS) + | set(economic_assumptions.YEAR_SPECIFIC_INPUTS) + | set(economic_assumptions.BEREAVEMENT_SPLIT_INPUTS) + | set(economic_assumptions.RESIDENTIAL_PROPERTY_INPUTS) + ) + for name, series in declared.items(): + for column in {name, stored_as.get(name, name)}: + listed = {index for index, names in uprating.items() if column in names} + assert listed == {series}, column + assert column not in custom, column + + resolver = UKMeasureResolver( + simulation_source=None, + frame=_frame(**_DESCRIPTORS), + scratch_dir=tmp_path, + year=2025, + ) + resolver.compute("person", "employment_income_annual") + factors = resolver.receipt()["annual_income_measures"][ + "other_spells_engine_factors" + ] + growth = resolver.simulation.tax_benefit_system.parameters.get_child( + declared["employment_income_before_lsr"] + ) + assert factors == {"employment_income_before_lsr": 1.0 + float(growth("2025"))} + pay, _ = resolver.compute("person", "employment_income") + np.testing.assert_allclose( + np.asarray(pay, dtype=float)[0] / 30_000.0, + factors["employment_income_before_lsr"], + rtol=1e-6, + ) + + def test_the_income_tax_rows_read_the_tax_before_the_winter_fuel_charge( tmp_path, ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py index 6a762e5b93..cdcc5b011c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py @@ -1963,6 +1963,53 @@ def __init__(self, *, dataset): ) +def test_the_other_spells_factor_is_the_engines_growth_parameters( + monkeypatch, tmp_path +): + """The factor the other spells grow by is the product of the engine's own + rates in the input's growth series over the years after the dataset's, + not a statistic of whichever block ran (microcosm#931); the engine's rows + must agree with it.""" + + rates = {2023: 0.06, 2024: 0.04, 2025: 0.03} + then = np.array([100.0, 0.0, 50.0]) + factor = (1 + rates[2023]) * (1 + rates[2024]) * (1 + rates[2025]) + arrays = {2022: then, 2025: then * factor * (1 + 4e-6)} + resolver = UKMeasureResolver.__new__(UKMeasureResolver) + resolver.year = 2025 + resolver.frame = SimpleNamespace(metadata={"time_period": "2022"}) + resolver._annual_income_factors = {} + read = [] + + def get_child(path): + read.append(path) + return lambda year: rates[int(year)] + + resolver.simulation = SimpleNamespace( + tax_benefit_system=SimpleNamespace( + parameters=SimpleNamespace(get_child=get_child) + ), + calculate=lambda name, year: arrays[int(year)], + ) + # Every carrier is off the parameter by 4e-6, inside the tolerance: the + # factor is the parameter, which a median of the rows would not be. + assert resolver._engine_projection_factor("employment_income_before_lsr") == ( + pytest.approx(factor, rel=1e-15) + ) + assert read == ["gov.economic_assumptions.yoy_growth.obr.average_earnings"] + assert resolver._annual_income_factors == { + "employment_income_before_lsr": pytest.approx(factor, rel=1e-15) + } + arrays[2025] = then * factor * (1 + 2e-5) + with pytest.raises(ValueError, match="by one factor"): + resolver._engine_projection_factor("self_employment_income") + with pytest.raises(ValueError, match="No engine growth series"): + resolver._engine_projection_factor("private_pension_income") + resolver.year = 2021 + with pytest.raises(ValueError, match="cannot be carried back"): + resolver._engine_projection_factor("employment_income_before_lsr") + + def _bus_frame() -> FrameStub: """Three households: North East, London and Wales, with NTS trips.""" @@ -2485,6 +2532,21 @@ def __init__(self, descriptors=_ANNUAL_DESCRIPTORS): self._tables["person"][column] = values +class _AnnualGrowthParameters: + """The stub engine's parameters: only the growth series its projection + carries pay and profit by (no dated tree, so no bus pricing).""" + + def __init__(self, engine): + self._engine = engine + + def get_child(self, path): + variable = self._engine.growth_series[path] + return lambda year: self._engine.uprating[variable] - 1.0 + + def __call__(self, period): + raise AttributeError("the stub engine carries no dated parameters") + + class _AnnualSimulation: """A stub engine that uprates stored inputs and taxes total income.""" @@ -2497,6 +2559,17 @@ class _AnnualSimulation: "esa_contrib_reported": 1.02, "incapacity_benefit_reported": 1.02, } + #: The growth series the stub's projection carries pay and profit by, as + #: policyengine-uk's parameters hold them: the other spells grow by the + #: series' rate (microcosm#931). + growth_series = { + "gov.economic_assumptions.yoy_growth.obr.average_earnings": ( + "employment_income_before_lsr" + ), + "gov.economic_assumptions.yoy_growth.obr.per_capita.mixed_income": ( + "self_employment_income" + ), + } extra_inputs: dict = {} def __init__(self, *, dataset): @@ -2539,7 +2612,9 @@ def __init__(self, *, dataset): variables["tax_band"] = SimpleNamespace( entity=SimpleNamespace(key="person"), value_type=str ) - self.tax_benefit_system = SimpleNamespace(variables=variables) + self.tax_benefit_system = SimpleNamespace( + variables=variables, parameters=_AnnualGrowthParameters(type(self)) + ) def set_input(self, variable, year, values): self.stored[(variable, int(year))] = np.asarray(values, dtype=float) From 3d5ad699f896640d5ab2f5e3c9a1d255b7572db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:30:39 +0000 Subject: [PATCH 003/108] Correct each engine block to the pool's share denominator and judge the whole-population formulas it runs (#931) After the move the dense measure step's per-clone engine blocks are no longer identical copies, so policyengine-uk's share formulas (shareholding, corporate_land_value, consumption_shareholding) would read a block's denominator instead of the pool's. - The shared engine_population module gains EngineBlock, the life of one block on its own engine. It validates the declaration, refuses the position window and periods before the data, and runs checks (a) and (b). It sets each share formula to its own value times c_b = f_b * S_b / S in the simulation and its baseline, and arms the formula-run recorder. Every simulation built later gets the same arrays and recorder. At the end it checks that each still holds the block's keys and weights, then judges them all. A record pass records refusals and a raise pass raises; work after the verdict is refused. The country passes its declaration and a pass. validate_pool_shares and PopulationDeclaration.correction_reads come with it. - UKMeasureResolver takes engine_pool_shares (scratch mode only) and opens an EngineBlock with the UK declaration and the measure pass, which records and never raises. A sibling that would set a key, its inputs or the weights is refused. close_engine_block judges the block. - full_measure builds each block's pool shares from the totals the representation already records and passes them to the block's resolver. After the blocks run, exact holds only if every block validated the declaration, passed both checks, applied the correction in every simulation it built, kept its keys and weights to the end, agreed with the others on its growth and annual-income factors, and got a clean verdict. exact_basis names this basis. exact_scope says float32-level, not bitwise, and names the Universal Credit rebalancing draw the window refuses. The identical-copies checks stay as diagnostics. block_sensitivity matches the land rows' engine variables, so present_in_this_run is no longer always empty. - The two frame modules join UK_TARGET_COMPILE_MODULE_NAMES. The fake engine the shared tests use moves to test_support so the build tests can use it too. No data changes on today's lines. Identical copies take a correction of 1 and give the same matrix and local metrics, bitwise, as without it, and a single-block resolver records none of the new keys, so national receipts keep their bytes. Today's K=15 dense receipts change by design: - the representation gains exact_basis, share_correction, population_reach and annual_income_projection_factors; - exact_scope is rewritten, and the release manifest's engine_measurement limitation copies it; - block_sensitivity is rewritten; - each block's resolver receipt gains engine_pool_shares and engine_population_reach; - exact now also needs a clean guard verdict, a stricter release guard on that line. The full and national target kernels re-key; no spine stage does. On a synthetic pool of three clones that differ, three blocks match one block over the pool: household-local and annual rows and every local metric bitwise, and the land rows and each block's share formulas within the float32 bound. Uncorrected land rows miss by more than 1%, and a decile read, a 2026 period and a counterfactual on corporate wealth are each refused. Engine-free tests run EngineBlock on fake engines. They drive checks (a) and (b), a sibling that moves its keys or weights, the window and a raise pass, and hold its receipts to the representation's finalisation. Co-Authored-By: Claude Opus 5.5 --- .../build/uk_runtime/full_measure.py | 395 +++++++++-- .../build/uk_runtime/measure_simulation.py | 115 ++- .../microcosm/build/uk_runtime/rowwise_cli.py | 19 +- .../uk_runtime/target_compile_modules.py | 5 + .../uk/test_uk_engine_block_correction.py | 586 ++++++++++++++++ .../uk/test_uk_weight_share_formula_inputs.py | 13 + .../engine_free/uk/test_uk_full_measure.py | 488 +++++++++++-- .../uk/test_uk_measure_simulation.py | 169 +++++ .../frame/adapters/engine_population.py | 341 ++++++++- .../shared/test_engine_population.py | 659 +++++++++--------- .../engine_population_fakes.py | 386 ++++++++++ 11 files changed, 2724 insertions(+), 452 deletions(-) create mode 100644 packages/microcosm-build/tests/engine/uk/test_uk_engine_block_correction.py create mode 100644 test_support/microcosm_frame/engine_population_fakes.py diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py index 7497b27ca6..36a1403610 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py @@ -7,6 +7,7 @@ from __future__ import annotations import gc +import math from collections.abc import Mapping from pathlib import Path from typing import Any @@ -29,24 +30,55 @@ from microcosm.build.uk_runtime.national_frame import uk_time_period from microcosm.calibrate.matrix import CalibrationProblem, build_constraint_matrix from microcosm.frame import Frame, MassChangeRecord +from microcosm.frame.adapters.engine_population import ( + MEASURE_PASS, + PoolShare, + PoolShareError, + factor_spread, + pool_shares, +) +from microcosm.frame.adapters.policyengine_uk_population import UK_ENGINE_POPULATION + +#: The engine variables the national targets bind that read policyengine-uk's +#: share formulas (``ons.land.corporate_land_value`` and ``ons.land.land_value``), +#: matched against the run's measure inputs. +UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ("corporate_land_value", "land_value") -UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ( - "ons/corporate_land_value", - "ons/land_value", - "slc/student_loan_repayment/england", +#: What ``engine_population_representation.exact`` certifies for per-clone +#: engine blocks (María's ruling of 2026-11-06, microcosm#931). A receipt +#: without it predates the correction, when exact meant identical copies. +UK_ENGINE_BLOCK_EXACT_BASIS = "pool_share_injection_with_population_guard" +UK_ENGINE_BLOCK_EXACT_SCOPE = ( + "per-clone engine blocks stand for the pool: each block's household weights " + "are scaled to the pool's mass, and in every simulation the block builds " + "shareholding, corporate_land_value and consumption_shareholding are set at " + "each period it computes to the block's own value times its pool-mass " + "factor times its allocation-key total over the pool's, the engine carrying " + "each key and the weights by one factor per year (checked per block); " + "months_since_last_birthday and attends_private_school are stored inputs, " + "every other policyengine-uk whole-population formula that ran was zero " + "under the parameters checked, and nothing the block computed fell in the " + "Universal Credit rebalancing years (2026 on), where policyengine-uk draws " + "new-claimant status by benefit-unit position; the result equals one " + "engine over the pool to float32 rounding, not bitwise, at the measure " + "step's starting weights, and the annual-income simulation carries other " + "spells by the engine's own growth parameters, the same in every block" ) +#: How far the blocks' parameter-driven factors (the engine's growth of each +#: key and of the weights, and the annual-income factor) may spread, relative +#: to their mean: float32 rounding, far inside it. +_BLOCK_FACTOR_AGREEMENT = 1e-6 #: policyengine-uk's weight-share formulas (a national total or a share #: allocated by ``x * w / sum(x * w)``) and the frame columns their allocation #: key ``x`` reads (microcosm#1115 review). Scaling a block's engine weights by -#: pool mass / block mass makes such a formula exact only when every block -#: carries the same ``sum(x * w)``; identical (source household, weight) -#: multisets guarantee that only while ``x`` is copied into every clone, which -#: holds for these keys (spine wealth and consumption inputs) and would not for -#: anything drawn per clone. The representation check therefore records each -#: block's ``sum(x * w)`` for every key here and requires them to agree; a -#: formula of this shape that reads a per-clone input must be added here so -#: the check sees it, never assumed exact. +#: pool mass / block mass leaves such a formula exact only while every block +#: carries the same ``sum(x * w)``, which clones that differ do not. The +#: representation therefore records each block's ``sum(x * w)`` for every key +#: here, and each block's resolver sets the formula to its own value times +#: ``f_b * S_b / S`` (microcosm#931). The columns are the adds the UK +#: declaration names for each key (``UK_ENGINE_POPULATION.key_adds``), which a +#: test holds them to, and which the engine's own adds are held to. UK_WEIGHT_SHARE_FORMULA_INPUTS: Mapping[str, tuple[str, ...]] = { # corporate_land_value and shareholding allocate by corporate_sector_wealth. "corporate_land_value": ("corporate_wealth", "private_pension_wealth"), @@ -171,17 +203,19 @@ def _engine_block_frames(frame, blocks: int) -> list[tuple[int | None, Frame]]: def _engine_population_representation(frame, block_frames) -> dict[str, Any]: - """How each engine block stands for the pool, and whether exactly. + """How each engine block stands for the pool, before the blocks run. A weight-share formula (a national total allocated by ``x*w / sum(x*w)``, policyengine-uk's corporate land value among them) sees only its block's denominator, so each block would reproduce the whole national total: the ``K`` times artefact behind the #736 erratum. Every block's engine weights - are therefore scaled by ``pool mass / block mass``. For blocks that are - identical copies of one another (the clone expansion's contract) every - weighted sum then equals the pool's and the formula is exact; the checks - record whether the blocks are such copies. A pool that is not leaves the - scaling approximate, and the measures receipt keeps its caveat. + are therefore scaled by ``pool mass / block mass``, and each block's + resolver corrects the share formulas to the pool's denominator from the + allocation-key totals recorded here (:func:`_block_pool_shares`). Whether + that made the blocks exact is known only once they ran: ``exact`` is + provisional until :func:`_finalize_representation`. The checks of whether + the blocks are identical copies stay as diagnostics; a block that lacks a + key column gets no correction, and the representation stays inexact. """ if len(block_frames) == 1 and block_frames[0][0] is None: @@ -209,11 +243,14 @@ def _engine_population_representation(frame, block_frames) -> dict[str, Any]: "represent the pool." ) factors[str(clone_index)] = pool_mass / block_mass - # Blocks are compared as multisets of (source household, weight): the - # clone expansion copies every spine household into every clone, so - # identical copies have identical multisets whatever geography each - # clone drew. Without a source identity the comparison is weights - # only, which cannot establish copies, so the result is never exact. + # A diagnostic (microcosm#931): blocks are compared as multisets of + # (source household, weight). The clone expansion copies every spine + # household into every clone, so identical copies have identical + # multisets whatever geography each clone drew. Without a source + # identity the comparison is weights only, which cannot establish + # copies, so the identical-copies diagnostic never holds. Whether the + # blocks stand for the pool is decided once they ran + # (_finalize_representation). key = next((c for c in _SOURCE_IDENTITY_COLUMNS if c in household), None) if key is None: identity_present = False @@ -275,29 +312,25 @@ def _engine_population_representation(frame, block_frames) -> dict[str, Any]: } return { "mode": "block_weights_scaled_to_pool", - "exact": bool( - identity_present - and identical - and equal_households - and equal_persons - and inputs_match - ), + # Provisional: _finalize_representation decides it from what every + # block's resolver recorded. + "exact": False, + "exact_basis": UK_ENGINE_BLOCK_EXACT_BASIS, "blocks": blocks, - # What ``exact`` certifies, and what it does not: on a slice of the - # K=25 pool, per-clone and single-block resolution matched on every - # household column and constituency metric but differed on a small - # share of person records (state pension type, pension credit, SLC - # repayment), so engine draws that depend on a record's position in - # the simulation are outside it (microcosm#1124). A dense release - # states this in its engine_measurement limitation; re-measuring the - # released file on one engine is the incumbent-surface evaluation's, - # an opt-in diagnostic since microcosm#1301. - "exact_scope": ( - "weight-share formulas see the pool's denominator in every block: " - "identical clone copies, pool-scaled block weights and equal " - "allocation-key sums; engine draws that depend on a record's " - "position in the simulation are not covered" - ), + # What ``exact`` certifies, and what it does not. A slice of the K=25 + # pool once differed between per-clone and single-block resolution on + # a few person records (microcosm#1124), through draws a record's + # position or a block's own identifiers decided. The identifiers are + # the release's since microcosm#1194; the one draw policyengine-uk + # still makes by position, which Universal Credit claimants count as + # new claimants in the rebalancing years from 2026 + # (scenarios/uc_reform.py, REBALANCING_YEARS), is outside it, and the + # guard refuses any block work in those years rather than certify it. + # A dense release states this in its engine_measurement limitation; + # re-measuring the released file on one engine is the + # incumbent-surface evaluation's, an opt-in diagnostic since + # microcosm#1301. + "exact_scope": UK_ENGINE_BLOCK_EXACT_SCOPE, "factor_by_block": factors, "checks": { "identity_key": identity_key, @@ -314,6 +347,227 @@ def _engine_population_representation(frame, block_frames) -> dict[str, Any]: } +def _block_pool_shares( + representation: Mapping[str, Any], +) -> dict[str, dict[str, PoolShare]] | None: + """Each block's :class:`PoolShare` for every share formula, or None. + + From the representation's own frame totals: ``S_b``, each block's + ``sum(x * w)`` at unscaled weights, the pool's ``S`` as their sum, and the + block's pool-mass factor ``f_b``, so ``c_b = f_b * S_b / S``. None for a + single block, and when a block lacks a key column or a total is not + finite: no block is then corrected, and the representation stays inexact. + """ + + if representation.get("mode") != "block_weights_scaled_to_pool": + return None + factors = representation["factor_by_block"] + inputs = representation["checks"]["weight_share_inputs"] + formulas = dict(UK_ENGINE_POPULATION.corrected) + if set(inputs) != set(formulas): + return None + for formula in formulas: + entry = inputs[formula] + if entry["missing_columns"] or set(entry["sum_by_block"]) != set(factors): + return None + try: + return pool_shares( + { + block: { + formula: float(inputs[formula]["sum_by_block"][block]) + for formula in formulas + } + for block in factors + }, + {block: float(factor) for block, factor in factors.items()}, + key_variables=formulas, + ) + except PoolShareError: + return None + + +def _finite_or_none(value: float) -> float | None: + return float(value) if math.isfinite(value) else None + + +def _finalize_representation( + representation: Mapping[str, Any], + resolver_receipts: list[Mapping[str, Any]], + *, + shares_built: bool, +) -> dict[str, Any]: + """Whether the per-clone engine blocks stood for the pool, once they ran. + + Read from each block's resolver receipt (``engine_pool_shares`` and + ``engine_population_reach``). ``exact`` holds only if every block's + resolver validated the declaration, passed checks (a) and (b), applied + the correction in every simulation it built, refused nothing, and still + held the block's keys and weights in each at the end; the blocks computed + at the same periods; their growth factors and annual-income factors + agree within ``_BLOCK_FACTOR_AGREEMENT``; and every block's verdict under + the measure pass is clean, so nothing reached a refused rank or median, + nothing a stored input stands for was derived, and nothing ran in the + rebalancing years. A single block is returned as it is. + """ + + finalized = dict(representation) + if representation.get("mode") != "block_weights_scaled_to_pool": + return finalized + blocks = [str(block) for block in representation["factor_by_block"]] + receipts = list(resolver_receipts) + by_block = dict(zip(blocks, receipts, strict=False)) + complete = shares_built and len(receipts) == len(blocks) + shares = { + block: receipt.get("engine_pool_shares") for block, receipt in by_block.items() + } + shares = { + block: entry for block, entry in shares.items() if isinstance(entry, Mapping) + } + complete = complete and len(shares) == len(blocks) + + def every(field: str) -> bool: + return complete and all(entry.get(field) is True for entry in shares.values()) + + periods = {tuple(entry.get("periods") or ()) for entry in shares.values()} + growth = { + block: dict(entry.get("growth_factors") or {}) + for block, entry in shares.items() + } + growth_periods = sorted( + {period for factors in growth.values() for period in factors} + ) + growth_complete = all( + set(factors) == set(growth_periods) for factors in growth.values() + ) + growth_spread = 0.0 + for period in growth_periods: + spread = factor_spread( + { + block: factors[period] + for block, factors in growth.items() + if period in factors + } + ) + growth_spread = max(growth_spread, *spread.values(), 0.0) + growth_agree = growth_complete and growth_spread <= _BLOCK_FACTOR_AGREEMENT + annual = { + block: dict( + (receipt.get("annual_income_measures") or {}).get( + "other_spells_engine_factors" + ) + or {} + ) + for block, receipt in by_block.items() + } + annual_spread = max(factor_spread(annual).values(), default=0.0) + annual_agree = annual_spread <= _BLOCK_FACTOR_AGREEMENT + + verdicts = { + block: receipt.get("engine_population_reach") + for block, receipt in by_block.items() + } + verdicts = { + block: verdict + for block, verdict in verdicts.items() + if isinstance(verdict, Mapping) + } + clean = ( + complete + and len(verdicts) == len(blocks) + and all( + verdict.get("clean") is True and verdict.get("pass") == MEASURE_PASS.name + for verdict in verdicts.values() + ) + ) + digests = {verdict.get("declaration_sha256") for verdict in verdicts.values()} + engines = { + tuple(sorted(dict(verdict.get("engine") or {}).items())) + for verdict in verdicts.values() + } + classes: dict[str, set[str]] = {} + refused: set[str] = set() + window_violations: dict[tuple[str, str, str], dict[str, Any]] = {} + for verdict in verdicts.values(): + for name, members in dict(verdict.get("classes") or {}).items(): + classes.setdefault(str(name), set()).update(map(str, members)) + refused.update(map(str, verdict.get("refused") or ())) + for violation in dict(verdict.get("window") or {}).get("violations") or (): + violation = dict(violation) + key = tuple( + str(violation.get(field)) for field in ("kind", "name", "period") + ) + window_violations[key] = violation + refusals = { + block: list(entry["refusals"]) + for block, entry in shares.items() + if entry.get("refusals") + } + exact = bool( + complete + and every("declaration_valid") + and every("applied") + and every("key_sums_verified") + and every("uniform_growth_verified") + and every("siblings_match") + and not refusals + and len(periods) == 1 + and growth_agree + and annual_agree + and clean + and len(digests) == 1 + and not window_violations + ) + finalized["exact"] = exact + finalized["share_correction"] = { + "formulas": dict(UK_ENGINE_POPULATION.corrected), + "keys_complete": bool(shares_built), + "periods": sorted({period for entry in periods for period in entry}), + "corrections_by_block": { + block: dict(entry.get("corrections") or {}) + for block, entry in shares.items() + }, + "applied_in_every_block": every("applied"), + "key_sums_verified": every("key_sums_verified"), + "largest_key_sum_gap": max( + ( + float(entry["largest_key_sum_gap"]) + for entry in shares.values() + if entry.get("largest_key_sum_gap") is not None + ), + default=None, + ), + "uniform_growth_verified": every("uniform_growth_verified"), + "growth_factors_by_block": growth, + "growth_factors_agree_across_blocks": growth_agree, + "max_growth_factor_spread": _finite_or_none(growth_spread), + "siblings_match": every("siblings_match"), + "refusals_by_block": refusals, + } + finalized["population_reach"] = { + "pass": MEASURE_PASS.name, + "declaration_valid": every("declaration_valid"), + "declaration_sha256": next(iter(digests)) if len(digests) == 1 else None, + "engine": dict(next(iter(engines))) if len(engines) == 1 else None, + "clean_in_every_block": clean, + "classes": {name: sorted(members) for name, members in sorted(classes.items())}, + "refused": sorted(refused), + "window": { + "first_year": UK_ENGINE_POPULATION.window.first_year, + "violations": [window_violations[key] for key in sorted(window_violations)], + }, + "violations_by_block": { + block: list(verdict.get("violations") or ()) + for block, verdict in verdicts.items() + if verdict.get("clean") is not True + }, + } + finalized["annual_income_projection_factors"] = { + "agree_across_blocks": annual_agree, + "max_rel_spread": _finite_or_none(annual_spread), + } + return finalized + + def _run_engine_blocks( block_frames, national_registry, @@ -324,6 +578,7 @@ def _run_engine_blocks( local_grains: tuple[str, ...], on_block, engine_weight_scales: Mapping[str, float] | None = None, + engine_pool_shares: Mapping[str, Mapping[str, PoolShare]] | None = None, identifier_universe=None, ) -> tuple[ dict[str, list[pd.DataFrame]], @@ -352,6 +607,12 @@ def _run_engine_blocks( resolver_kwargs["engine_weight_scale"] = float( engine_weight_scales[str(clone_index)] ) + block_shares = None + if engine_pool_shares is not None and clone_index is not None: + # The block's share formulas take the pool's denominator in every + # simulation its resolver builds (microcosm#931). + block_shares = engine_pool_shares[str(clone_index)] + resolver_kwargs["engine_pool_shares"] = block_shares if identifier_universe is not None and clone_index is not None: # A block's rows take the identifiers they have in the release, # which numbers the whole frame (microcosm#1194). @@ -385,6 +646,12 @@ def _run_engine_blocks( input_period=int(uk_time_period(block_frame)), ) ) + if block_shares is not None: + # Everything the block computes is computed: judge it while its + # engine is alive, so the receipt below carries the verdict. + close = getattr(resolver, "close_engine_block", None) + if callable(close): + close() resolver_receipts.append( _without_scratch_paths(dict(resolver.receipt()), scratch_dir) ) @@ -479,37 +746,45 @@ def _measures_receipt( receipt["engine_population_representation"] = dict(representation) exact = bool(representation and representation.get("exact")) receipt["deviation"] = "per_clone_block_engine_resolution" + # The engine variables the measures read, matched by name: the + # measure inputs are keyed (entity, engine variable). + measured = {variable for _, variable in national_input_keys} present = sorted( column for column in UK_BLOCK_SENSITIVE_MEASURE_COLUMNS - if column in {variable for _, variable in national_input_keys} + if column in measured ) receipt["block_sensitivity"] = { "known_population_normalised_measures": list( UK_BLOCK_SENSITIVE_MEASURE_COLUMNS ), "present_in_this_run": present, + "corrected_formulas": dict(UK_ENGINE_POPULATION.corrected), "weight_share_formulas": { formula: list(columns) for formula, columns in UK_WEIGHT_SHARE_FORMULA_INPUTS.items() }, "mitigation": ( "each block's engine household weights are scaled by pool mass / " - "block mass, so weight-share formulas see the pool's denominator; " - "exact only while the blocks are identical copies and every " - "allocation key named in weight_share_formulas carries the same " - "sum(x * w) in every block" + "block mass, and every simulation the block builds sets the " + "corrected formulas, at each period it computes, to the block's " + "own value times its pool-mass factor times its allocation-key " + "total over the pool's, so they see the pool's denominator; " + "exact only when every block applied the correction with its " + "checks holding and every whole-population formula it ran is one " + "the declaration allows (engine_population_representation)" ), "exact": exact, "caveat": ( None if exact else ( - "the engine blocks are not identical copies of one another, so " - "the scaling is approximate and per-block engine resolution " - "may mis-measure population-normalised formulas; rows on these " - "measures are not evidence for adjudication from this run. " - "Resolve in a single block before ruling on them." + "the engine blocks are not certified to stand for the pool " + "(see engine_population_representation), so per-block engine " + "resolution may mis-measure population-normalised formulas; " + "rows on these measures are not evidence for adjudication " + "from this run. Resolve in a single block before ruling on " + "them." ) ), } @@ -547,6 +822,7 @@ def resolve_uk_full_measures( block_frames = _engine_block_frames(frame, blocks) representation = _engine_population_representation(frame, block_frames) + block_shares = _block_pool_shares(representation) measure_parts: dict[tuple[str, str], list[pd.Series]] = {} def collect(block_frame, block_inputs): @@ -567,10 +843,14 @@ def collect(block_frame, block_inputs): scratch_dir=scratch_dir, resolver_factory=resolver_factory, engine_weight_scales=representation.get("factor_by_block"), + engine_pool_shares=block_shares, identifier_universe=frame, local_grains=local_grains, on_block=collect, ) + representation = _finalize_representation( + representation, resolver_receipts, shares_built=block_shares is not None + ) measure_inputs: dict[tuple[str, str], np.ndarray] = {} for (entity, variable), parts in measure_parts.items(): @@ -767,6 +1047,7 @@ def resolve_uk_full_national_problem( block_frames = _engine_block_frames(frame, blocks) representation = _engine_population_representation(frame, block_frames) + block_shares = _block_pool_shares(representation) targets = national_registry.to_target_set() band_edges = national_registry if band_edge_registry is None else band_edge_registry block_problems: list[tuple[CalibrationProblem, np.ndarray]] = [] @@ -795,11 +1076,15 @@ def materialize(block_frame, block_inputs): scratch_dir=scratch_dir, resolver_factory=resolver_factory, engine_weight_scales=representation.get("factor_by_block"), + engine_pool_shares=block_shares, identifier_universe=frame, local_grains=local_grains, on_block=materialize, ) ) + representation = _finalize_representation( + representation, resolver_receipts, shares_built=block_shares is not None + ) if any(report != reports[0] for report in reports[1:]): raise RuntimeError( "per-clone national target materialization differs across blocks." diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py index 66542cadfe..8cdbe196d7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -5,7 +5,7 @@ import functools import json import warnings -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from datetime import date from importlib import resources as importlib_resources from pathlib import Path @@ -86,6 +86,12 @@ ) from microcosm.calibrate import TargetRegistry from microcosm.frame import Frame, MassChangeRecord, Weights +from microcosm.frame.adapters.engine_population import ( + MEASURE_PASS, + EngineBlock, + PoolShare, + validate_pool_shares, +) from microcosm.frame.adapters.policyengine_uk import ( frame_isa_income_contract, uk_dataset_through, @@ -93,6 +99,7 @@ uk_whole_rebalancing_years, validate_uc_claimant_input, ) +from microcosm.frame.adapters.policyengine_uk_population import UK_ENGINE_POPULATION _ENTITY_LINK = {"benunit": "person_benunit_id", "household": "person_household_id"} _ENTITY_ID = { @@ -934,9 +941,11 @@ def _engine_scratch_frame( (and only those: the resolver's own frame keeps the block's true mass). A per-clone engine block carries a K-th of the pool, so a formula that allocates a national total by weighted share would hand every block the - whole total; scaled to the pool's mass the block's weighted sums equal - the pool's and the formula is exact for identical clone copies. The - scaling is declared on the engine frame's mass log. + whole total; scaled to the pool's mass each block hands out a share of it + instead. That share is the pool's for identical clone copies; for clones + that differ, the resolver's pool-share correction (``engine_pool_shares``) + sets each share formula to the pool's value. The scaling is declared on + the engine frame's mass log. The identifiers leave as the release's too (microcosm#1194, ``national_frame.uk_release_identifier_tables``): policyengine-uk stores @@ -1069,7 +1078,19 @@ def engine_bus_pricing_parameters(tax_benefit_system: Any, year: int) -> dict[st class UKMeasureResolver: - """B2 measure provider backed by a policyengine-uk Microsimulation.""" + """B2 measure provider backed by a policyengine-uk Microsimulation. + + ``engine_pool_shares`` makes the resolver one per-clone engine block of + the dense measure step (microcosm#931): a :class:`PoolShare` for each of + policyengine-uk's share formulas (``UK_ENGINE_POPULATION.corrected``), + carrying the block's ``engine_weight_scale``. Right after its simulation + is built the resolver opens the shared :class:`EngineBlock` on it, with + the UK declaration and the measure pass, which records and never raises + (María's ruling of 2026-11-06). Every simulation the resolver builds then + reads the pool's shares, and :meth:`receipt` says how; + :meth:`close_engine_block` judges what the block computed, once it has + computed everything. Scratch mode only. + """ def __init__( self, @@ -1081,11 +1102,25 @@ def __init__( microsimulation_factory: Any | None = None, engine_weight_scale: float | None = None, engine_identifier_universe: Any | None = None, + engine_pool_shares: Mapping[str, PoolShare] | None = None, ): self.year = int(year) self._engine_weight_scale = ( None if engine_weight_scale is None else float(engine_weight_scale) ) + self._engine_pool_block: EngineBlock | None = None + if engine_pool_shares is not None: + # A block's shares carry the factor its engine weights do (1 when + # unscaled); checked before any engine is built. + validate_pool_shares( + UK_ENGINE_POPULATION, + engine_pool_shares, + weight_scale=( + 1.0 + if self._engine_weight_scale is None + else self._engine_weight_scale + ), + ) policyengine_uk = _policyengine_uk_module() factory = ( microsimulation_factory @@ -1118,6 +1153,11 @@ def __init__( "an identifier universe needs scratch-mode resolution; a direct " "H5 source carries its own identifiers." ) + if engine_pool_shares is not None: + raise ValueError( + "pool shares need scratch-mode resolution; a direct H5 source " + "carries its own weights and is no engine block of a pool." + ) dropped = () source_path = Path(simulation_source) mode = "direct_h5" @@ -1179,6 +1219,9 @@ def __init__( validate_uc_claimant_input(self.simulation, frame.table("person"), self.year) self.contract_targets = _uk_contract_targets() bound_cgt_periods = {} + # The years a dated measure is computed at besides this one: an engine + # block's shares are corrected at each (microcosm#931). + bound_years: set[int] = set() for target_id, target in self.contract_targets.items(): binding = target["bindings"]["policyengine"] for name in _binding_variable_names(binding): @@ -1191,6 +1234,8 @@ def __init__( raise ValueError( f"company car benefit year/period mismatch: {target_id}" ) + if benefit_year is not None: + bound_years.add(int(benefit_year)) cgt_measure = _cgt_model_measure(name, self.year) if cgt_measure is None: continue @@ -1203,6 +1248,7 @@ def __init__( "model_variable": model_variable, "measurement_period": measure_year, } + bound_years.add(int(measure_year)) self._receipt = { "mode": mode, "source_path": str(source_path), @@ -1240,6 +1286,47 @@ def __init__( self._annual_income_fallback: str | None = None self._annual_income_simulation_cache: Any | None = None self._annual_income_factors: dict[str, float] = {} + if engine_pool_shares is not None: + # After the simulation is built and its roles validated, before + # anything reads it: the dataset's year, this year and every + # dated measure's year are the periods the block computes at. + dataset_year = self._dataset_year() + self._engine_pool_block = EngineBlock( + self.simulation, + engine_pool_shares, + declaration=UK_ENGINE_POPULATION, + policy=MEASURE_PASS, + periods=(dataset_year, self.year, *bound_years), + dataset_year=dataset_year, + ) + self._factory = self._engine_pool_block.wrap(self._factory) + + def close_engine_block(self) -> None: + """Judge what this engine block's simulations computed (microcosm#931). + + The dense measure step calls it once the block has resolved every + measure and local metric, before reading :meth:`receipt`; the + receipt then carries the verdict (``engine_population_reach``). A + resolver that is no engine block has nothing to judge. + """ + + block = getattr(self, "_engine_pool_block", None) + if block is not None: + block.close() + + def _note_engine_request(self, variable: str) -> None: + block = getattr(self, "_engine_pool_block", None) + if block is not None: + block.note_request(variable, self.year) + + def _refuse_pool_share_substitution( + self, names: Iterable[str | None], route: str + ) -> None: + """Refuse a sibling that would change what the block's correction read.""" + + block = getattr(self, "_engine_pool_block", None) + if block is not None: + block.refuse_substitution(names, route) def use_current_income_for_annual_measures(self, *, reason: str) -> None: """Let a frame without annual income descriptors read current values. @@ -1406,6 +1493,7 @@ def _refuse_dated_measure_after_window(self, variable: str) -> None: def compute(self, entity: str, variable: str) -> tuple[np.ndarray, str]: self._refuse_dated_measure_after_window(variable) + self._note_engine_request(variable) if variable in UK_ANNUAL_INCOME_MEASURES: return self._annual_income_measure(entity, variable) if variable in UK_HOUSEHOLD_ENERGY_MEASURES: @@ -1463,6 +1551,7 @@ def counterfactual_delta( ): raise ValueError(f"unsupported output_delta {direction!r}") metric = str(binding.get("metric_name") or output) + self._note_engine_request(output) allocation = binding.get("allocation") if allocation is not None: rate_class = _declared_rate_class(allocation, output, direction, entity) @@ -1733,6 +1822,15 @@ def _annual_income_simulation(self) -> Any: if self._annual_income_simulation_cache is not None: return self._annual_income_simulation_cache + self._refuse_pool_share_substitution( + ( + *(name for name, _share, _other in _ANNUAL_INCOME_INPUTS), + "state_pension", + _COMPANY_CAR_AVAILABILITY, + *_ANNUAL_INCOME_HELD_VARIABLES, + ), + "the annual-income simulation", + ) person = self.frame.table("person") simulation = self._new_simulation() annual_inputs: dict[str, np.ndarray] = {} @@ -1836,6 +1934,7 @@ def _engine_projection_factor(self, name: str) -> float: return factor def _counterfactual_simulation(self, zeroed: str, folded: str | None) -> Any: + self._refuse_pool_share_substitution((zeroed, folded), "a counterfactual") key = (zeroed, folded) if key in self._counterfactuals: return self._counterfactuals[key] @@ -1942,6 +2041,12 @@ def receipt(self) -> dict[str, Any]: } if getattr(self, "_annual_income_measures_used", None): receipt["annual_income_measures"] = self._annual_income_receipt() + block = getattr(self, "_engine_pool_block", None) + if block is not None: + # Only an engine block of the dense measure step carries these + # (engine_pool_shares, and engine_population_reach once judged), + # so a single-block receipt keeps its bytes (microcosm#931). + receipt.update(block.receipt_entries()) return receipt def _annual_income_receipt(self) -> dict[str, Any]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py index 8ec6c77892..9df503c43a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py @@ -1367,13 +1367,18 @@ def release_verdict( normalised formulas (a national total allocated by weighted share) saw one block's denominator and each block reproduced the whole aggregate, the ``K`` times land-value artefact behind the #736 erratum. The measures loop - now scales every block's engine weights to the pool and records whether the - blocks are identical copies with identical allocation keys for the named - weight-share formulas (``engine_population_representation.exact``; the - formulas and their inputs are ``UK_WEIGHT_SHARE_FORMULA_INPUTS``); a single - block is exact by construction. ``engine_population_exact`` is that - record; left unset, a multi-block run is treated as inexact. The posture is - written beside the verdict so a reader sees which leg failed. + now scales every block's engine weights to the pool, sets each block's + share formulas to the pool's value in every simulation it builds, and + judges every whole-population formula the block ran against the UK + declaration (``engine_population_representation.exact``, with + ``exact_basis`` ``pool_share_injection_with_population_guard``: corrected + blocks with their checks holding and a clean guard, equal to one engine + over the pool to float32 rounding; microcosm#931). A receipt without + ``exact_basis`` predates the correction, when exact meant blocks that were + identical copies with identical allocation keys. A single block is exact + by construction. ``engine_population_exact`` is that record; left unset, a + multi-block run is treated as inexact. The posture is written beside the + verdict so a reader sees which leg failed. """ single_block = int(engine_blocks) == 1 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py index bb6b495cbe..b832f200fd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py @@ -70,6 +70,11 @@ "microcosm.calibrate.hierarchy", "microcosm.calibrate.registry", "microcosm.calibrate.target", + # The dense measure step corrects each per-clone engine block to the + # pool's share denominator and judges the whole-population formulas it + # runs with these, against the UK declaration (microcosm#931). + "microcosm.frame.adapters.engine_population", + "microcosm.frame.adapters.policyengine_uk_population", ) EXCLUDED_DIRECT_IMPORTS: Mapping[str, str] = MappingProxyType( diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_engine_block_correction.py b/packages/microcosm-build/tests/engine/uk/test_uk_engine_block_correction.py new file mode 100644 index 0000000000..41ba3f00a4 --- /dev/null +++ b/packages/microcosm-build/tests/engine/uk/test_uk_engine_block_correction.py @@ -0,0 +1,586 @@ +"""Per-clone engine blocks corrected to the pool's share denominator (microcosm#931). + +The dense measure step resolves one engine block per clone, its household +weights scaled to the pool's mass. Once clones differ (in their keys, pay and +weights, and in which copies they hold), policyengine-uk's share formulas +would see a block's denominator, so each block's resolver sets them to the +pool's value in every simulation it builds and judges the whole-population +formulas the block ran. On a synthetic pool of three clones that differ, the +national problem and the local metrics resolved in three blocks are held to +one block over the pool: the household-local rows (the annual-income row +among them) and the local metrics bitwise, the land rows within the float32 +bound, and the representation exact on its new basis. Negative controls show +what the correction and the guard catch, and identical copies resolve +bitwise as they did without the correction. Every household here is made up. +""" + +from __future__ import annotations + +import math + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import full_measure +from microcosm.build.uk_runtime.full_measure import ( + resolve_uk_full_measures, + resolve_uk_full_national_problem, +) +from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column +from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.calibrate.matrix import build_constraint_matrix +from microcosm.frame import WeightKind +from microcosm.frame.adapters import engine_population +from microcosm.frame.adapters.engine_population import ( + pool_share_atol, + pool_share_rtol, +) +from microcosm.frame.adapters.policyengine_uk_population import UK_ENGINE_POPULATION + +K = 3 +SOURCES = 8 +D, P = 2024, 2025 +REGIONS = ("LONDON", "NORTH_WEST", "SCOTLAND", "WALES", "SOUTH_EAST") +COICOP = UK_ENGINE_POPULATION.key_adds["consumption"] +CLONE = ladder_clone_index_column("household") +#: The registry's rows: the two land rows, which read the corrected shares, +#: a household-local one, and an annual-income one (the annual simulation). +LAND = ("ons.land.corporate_land_value", "ons.land.land_value") +LOCAL = ("obr.ni_employee", "obr.income_tax") +#: Everything that reads a share formula: none may be known before the +#: shares are set. +SHARE_READERS = ( + "shareholding", + "corporate_land_value", + "consumption_shareholding", + "business_rates", + "corporate_sdlt", + "land_value", + "LVT", + "household_tax", + "household_net_income", + "gov_tax", +) + + +def _pool(*, differ: bool): + """Three clones of eight made-up households. With ``differ`` each clone + draws its own keys, pay and weights, and clone 1 holds a ninth household, + a copy only it carries (as an SPI copy would be).""" + + persons, benunits, households, weights = [], [], [], [] + for clone in range(K): + sources = [*range(SOURCES), *([99] if differ and clone == 1 else [])] + for slot, source in enumerate(sources): + household_id = (clone + 1) * 100 + slot + shared = np.random.default_rng(1000 + source) + drawn = np.random.default_rng(5000 + source + (37 * clone if differ else 0)) + weight = 1000.0 * (1 + source % 4) / K + if differ: + weight *= float(drawn.uniform(0.8, 1.2)) + weights.append(weight) + row = { + "household_id": household_id, + "household_source_id": source, + CLONE: clone, + "region": REGIONS[source % len(REGIONS)], + "council_tax": 1_500.0 + 100.0 * (source % 4), + "rent": 9_000.0 if source % 2 else 0.0, + "tenure_type": "RENT_PRIVATELY" if source % 2 else "OWNED_OUTRIGHT", + "main_residence_value": ( + 0.0 if source % 2 else 300_000.0 + 10_000.0 * source + ), + "corporate_wealth": ( + float(drawn.lognormal(10, 1.5)) if source % 2 == 0 else 0.0 + ), + "private_pension_wealth": ( + float(drawn.lognormal(11, 1.0)) if source % 3 else 0.0 + ), + } + for column in COICOP: + row[column] = float(drawn.uniform(300, 6_000)) + households.append(row) + people = 1 + source % 3 + benunits.append( + { + "benunit_id": household_id * 10, + "dependent_children": 1 if people == 3 else 0, + "is_married": people >= 2, + } + ) + for p in range(people): + age = (44, 41, 9)[p] + pay = np.random.default_rng( + 9000 + 10 * source + p + (11 * clone if differ else 0) + ) + works = age >= 18 and (source + p) % 3 != 0 + self_employed = p == 0 and source % 4 == 1 + persons.append( + { + "person_id": household_id * 10 + p, + "person_benunit_id": household_id * 10, + "person_household_id": household_id, + "age": age, + "is_benunit_head": p == 0, + "is_parent": people == 3 and p < 2, + "employment_income": ( + float(pay.uniform(0, 70_000)) if works else 0.0 + ), + "self_employment_income": ( + float(pay.uniform(2_000, 30_000)) if self_employed else 0.0 + ), + # Stored as the spine stores them (the root stage and + # stage 52). + "months_since_last_birthday": float(shared.integers(0, 12)), + "attends_private_school": False, + "attends_private_school_random_draw": float(shared.uniform()), + # Each adult's year: half-year entrants, and taxable + # other spells in every clone (job changers and a + # leaver, businesses), which the annual simulation + # carries by the engine's factor: in a block and in + # the pool alike only if the factor is the engine's + # own parameter, not a statistic of the rows. + "employment_income_annual_share": ( + 0.5 if p == 0 and source % 5 == 2 else 1.0 + ), + "employment_income_other_spells": ( + 8_000.0 + 4_000.0 * (source % 8) + if p == 0 and source % 2 + else 0.0 + ), + "self_employment_income_annual_share": 1.0, + "self_employment_income_other_spells": ( + 12_000.0 if self_employed else 0.0 + ), + "private_pension_income_annual_share": 1.0, + "contributory_benefits_annual_share": 1.0, + } + ) + return uk_national_frame( + person=pd.DataFrame(persons), + benunit=pd.DataFrame(benunits), + household=pd.DataFrame(households), + time_period=D, + household_weights=np.asarray(weights, dtype=float), + weight_kind=WeightKind.IMPORTANCE, + ) + + +def _registry() -> TargetRegistry: + def spec(target_id: str, entity: str, measure: str, family: str) -> TargetSpec: + return TargetSpec( + name=target_id, + entity=entity, + value=1.0e9, + measure=measure, + period=P, + source="synthetic", + family=family, + metadata={ + "contract_target_id": target_id, + "measure_kind": "prepared_column", + }, + ) + + return TargetRegistry( + [ + spec(LAND[0], "household", "ons/corporate_land_value", "ons_land"), + spec(LAND[1], "household", "ons/land_value", "ons_land"), + spec(LOCAL[0], "person", "obr/ni_employee", "obr"), + spec(LOCAL[1], "person", "obr/income_tax", "obr"), + ], + country="uk", + ) + + +def _without_correction(**kwargs): + """A resolver as it was before the correction: no pool shares.""" + + kwargs.pop("engine_pool_shares", None) + return UKMeasureResolver(**kwargs) + + +def _rows(problem) -> dict[str, np.ndarray]: + matrix = problem.matrix.toarray() + return {name.split("@")[0]: matrix[row] for row, name in enumerate(problem.names)} + + +def _largest_block(frame) -> int: + return int(frame.table("household")[CLONE].value_counts().max()) + + +def _land_rtol(frame) -> float: + """The float32 bound for corporate_land_value, one year from the data.""" + + key = UK_ENGINE_POPULATION.corrected["corporate_land_value"] + return pool_share_rtol( + _largest_block(frame), + frame.n("household"), + key_inputs=len(UK_ENGINE_POPULATION.key_adds[key]), + projection_years=P - D, + ) + + +@pytest.fixture(scope="module") +def differing(tmp_path_factory): + """The differing pool resolved in one block and in three, with the + resolvers each run built (kept, so their engines can be read after).""" + + frame = _pool(differ=True) + runs, resolvers = {}, {} + for blocks in (1, K): + built = resolvers.setdefault(blocks, []) + + def keep(*, _built=built, **kwargs): + resolver = UKMeasureResolver(**kwargs) + _built.append(resolver) + return resolver + + runs[blocks] = resolve_uk_full_national_problem( + frame, + _registry(), + period=P, + scratch_dir=tmp_path_factory.mktemp(f"differing-{blocks}"), + blocks=blocks, + resolver_factory=keep, + ) + return frame, runs, resolvers + + +def test_household_local_rows_and_local_metrics_do_not_depend_on_the_blocks( + differing, +): + _frame, runs, _resolvers = differing + (one, _, one_metrics, _), (three, _, three_metrics, _) = runs[1], runs[K] + assert one.names == three.names + np.testing.assert_array_equal( + one.initial_weights.values, three.initial_weights.values + ) + pool, blocked = _rows(one), _rows(three) + for target_id in LOCAL: + # Bitwise: income tax on each person's year included, whose other + # spells grow by the engine's own growth parameters in every block. + np.testing.assert_array_equal(blocked[target_id], pool[target_id]) + assert np.any(pool[target_id] != 0) + assert set(one_metrics) == set(three_metrics) == {"constituency", "la"} + for grain, metrics in one_metrics.items(): + pd.testing.assert_frame_equal(three_metrics[grain], metrics, check_exact=True) + + +def test_land_rows_are_one_engine_over_the_pool_to_float32_rounding(differing): + frame, runs, _resolvers = differing + pool, blocked = _rows(runs[1][0]), _rows(runs[K][0]) + rtol = _land_rtol(frame) + corporate = pool["ons.land.corporate_land_value"] + assert np.any(corporate != 0) + np.testing.assert_allclose( + blocked["ons.land.corporate_land_value"], corporate, rtol=rtol, atol=0 + ) + # land_value adds each household's own land value to its corporate share: + # bounded per row, by the share term and the additions. + atol = pool_share_atol( + np.abs(corporate), + np.abs(pool["ons.land.land_value"]) + 2 * np.abs(corporate), + share_rtol=rtol, + additions=2, + ) + gap = np.abs(blocked["ons.land.land_value"] - pool["ons.land.land_value"]) + assert np.all(gap <= atol) + + +def test_corrected_blocks_are_exact_and_say_on_what_basis(differing): + _frame, runs, _resolvers = differing + receipt = runs[K][3] + representation = receipt["engine_population_representation"] + assert representation["exact"] is True + assert representation["exact_basis"] == "pool_share_injection_with_population_guard" + assert representation["mode"] == "block_weights_scaled_to_pool" + assert representation["blocks"] == K + # The clones are not identical copies: the old test of exactness fails, + # and stays as a diagnostic. + checks = representation["checks"] + assert checks["equal_household_counts"] is False + assert checks["weight_share_inputs_match"] is False + correction = representation["share_correction"] + assert correction["applied_in_every_block"] is True + assert correction["key_sums_verified"] is True + assert correction["uniform_growth_verified"] is True + assert correction["growth_factors_agree_across_blocks"] is True + assert correction["siblings_match"] is True + assert correction["periods"] == [str(D), str(P)] + corrections = [ + block["corporate_land_value"] + for block in correction["corrections_by_block"].values() + ] + assert max(abs(c - 1.0) for c in corrections) > 1e-3 + reach = representation["population_reach"] + assert reach["clean_in_every_block"] is True + assert reach["refused"] == [] + assert reach["window"] == {"first_year": 2026, "violations": []} + assert reach["classes"]["corrected"] == sorted(UK_ENGINE_POPULATION.corrected) + assert set(reach["classes"]["stored"]) == { + "attends_private_school", + "months_since_last_birthday", + } + assert representation["annual_income_projection_factors"] == { + "agree_across_blocks": True, + "max_rel_spread": 0.0, + } + sensitivity = receipt["block_sensitivity"] + assert sensitivity["exact"] is True and sensitivity["caveat"] is None + assert sensitivity["present_in_this_run"] == ["corporate_land_value", "land_value"] + # Each block's own resolution receipt carries its correction; the + # single-block run's carries none of it. + for resolution in receipt["resolution"]: + assert resolution["provider"]["engine_pool_shares"]["applied"] is True + single = runs[1][3] + assert "engine_population_representation" not in single + for resolution in single["resolution"]: + assert "engine_pool_shares" not in resolution["provider"] + assert "engine_population_reach" not in resolution["provider"] + + +#: The corrected formulas and two values that multiply a share by a national +#: total, each with the allocation key its share divides by. +SHARE_VALUES = ( + ("shareholding", "corporate_sector_wealth"), + ("consumption_shareholding", "consumption"), + ("business_rates", "corporate_sector_wealth"), + ("corporate_sdlt", "corporate_sector_wealth"), +) + + +def test_every_corrected_formula_takes_the_pools_value_in_every_block(differing): + """Each block's correction is ``f_b * S_b / S`` for every share formula, + and in each block's engine the three share formulas, and what multiplies + a share by a national total, take the pool's value for every household + to float32 rounding (microcosm#931).""" + + _frame, runs, resolvers = differing + representation = runs[K][3]["engine_population_representation"] + factors = representation["factor_by_block"] + inputs = representation["checks"]["weight_share_inputs"] + corrections = representation["share_correction"]["corrections_by_block"] + assert set(corrections) == set(factors) + for block, by_formula in corrections.items(): + assert set(by_formula) == set(UK_ENGINE_POPULATION.corrected) + for formula, correction in by_formula.items(): + sums = inputs[formula]["sum_by_block"] + expected = factors[block] * sums[block] / math.fsum(sums.values()) + assert correction == pytest.approx(expected, rel=1e-12) + (pool,) = resolvers[1] + assert len(resolvers[K]) == K + pool_ids = np.asarray(pool.simulation.calculate("household_id", P)) + for name, key in SHARE_VALUES: + reference = pd.Series( + np.asarray(pool.simulation.calculate(name, P), dtype=np.float64), + index=pool_ids, + ) + assert np.any(reference.to_numpy() != 0), name + for resolver in resolvers[K]: + ids = np.asarray(resolver.simulation.calculate("household_id", P)) + values = np.asarray( + resolver.simulation.calculate(name, P), dtype=np.float64 + ) + rtol = pool_share_rtol( + len(ids), + len(pool_ids), + key_inputs=len(UK_ENGINE_POPULATION.key_adds[key]), + projection_years=P - D, + ) + np.testing.assert_allclose( + values, reference.loc[ids].to_numpy(), rtol=rtol, atol=0, err_msg=name + ) + + +def test_the_measures_entry_point_corrects_its_blocks_too(differing, tmp_path): + frame, runs, _resolvers = differing + registry = _registry() + prepared, _restore, _rows_, metrics, receipt = resolve_uk_full_measures( + frame, registry, period=P, scratch_dir=tmp_path, blocks=K + ) + assert receipt["engine_population_representation"]["exact"] is True + problem = build_constraint_matrix(prepared, registry.to_target_set(), "household") + pool, measured = _rows(runs[1][0]), _rows(problem) + for target_id in LOCAL: + np.testing.assert_array_equal(measured[target_id], pool[target_id]) + np.testing.assert_allclose( + measured["ons.land.corporate_land_value"], + pool["ons.land.corporate_land_value"], + rtol=_land_rtol(frame), + atol=0, + ) + for grain, values in runs[1][2].items(): + pd.testing.assert_frame_equal(metrics[grain], values, check_exact=True) + + +def test_without_the_correction_the_land_rows_miss_the_pool(differing, tmp_path): + frame, runs, _resolvers = differing + _problem, _rows_, _metrics, receipt = resolve_uk_full_national_problem( + frame, + _registry(), + period=P, + scratch_dir=tmp_path, + blocks=K, + resolver_factory=_without_correction, + local_grains=(), + ) + representation = receipt["engine_population_representation"] + assert representation["exact"] is False + assert representation["share_correction"]["applied_in_every_block"] is False + pool = _rows(runs[1][0])["ons.land.corporate_land_value"] + blocked = _rows(_problem)["ons.land.corporate_land_value"] + carried = pool != 0 + assert np.max(np.abs(blocked[carried] / pool[carried] - 1.0)) > 1e-2 + + +@pytest.fixture(scope="module") +def identical(tmp_path_factory): + frame = _pool(differ=False) + return { + label: resolve_uk_full_national_problem( + frame, + _registry(), + period=P, + scratch_dir=tmp_path_factory.mktemp(f"identical-{label}"), + blocks=K, + resolver_factory=factory, + ) + for label, factory in ( + ("corrected", UKMeasureResolver), + ("uncorrected", _without_correction), + ) + } + + +def test_identical_copies_resolve_bitwise_as_they_did_without_the_correction( + identical, +): + """Today's dense line: clones are identical copies, so every block's + correction is 1 and the arrays it sets are the engine's own, bitwise; the + matrix and the local metrics do not move (microcosm#931).""" + + corrected, uncorrected = identical["corrected"], identical["uncorrected"] + assert corrected[0].names == uncorrected[0].names + assert (corrected[0].matrix != uncorrected[0].matrix).nnz == 0 + np.testing.assert_array_equal( + corrected[0].target_vector, uncorrected[0].target_vector + ) + for grain, metrics in uncorrected[2].items(): + pd.testing.assert_frame_equal(corrected[2][grain], metrics, check_exact=True) + representation = corrected[3]["engine_population_representation"] + assert representation["exact"] is True + assert representation["checks"]["identical_source_weight_multisets"] is True + for block in representation["share_correction"]["corrections_by_block"].values(): + for correction in block.values(): + assert correction == pytest.approx(1.0, abs=1e-12) + + +def _block_resolver(frame, tmp_path, *, clone=0, year=P, resolver=UKMeasureResolver): + block_frames = dict(full_measure._engine_block_frames(frame, K)) + representation = full_measure._engine_population_representation( + frame, list(block_frames.items()) + ) + shares = full_measure._block_pool_shares(representation) + return resolver( + simulation_source=None, + scratch_dir=tmp_path, + year=year, + frame=block_frames[clone], + engine_weight_scale=representation["factor_by_block"][str(clone)], + engine_identifier_universe=frame, + engine_pool_shares=shares[str(clone)], + ) + + +def test_no_share_reader_is_known_before_the_shares_are_set(monkeypatch, tmp_path): + frame = _pool(differ=True) + known_before: list[dict[str, list]] = [] + inject = engine_population.inject_pool_shares + + def watching(simulation, shares, periods, *, weight_variable): + names = [ + name + for name in SHARE_READERS + if name in simulation.tax_benefit_system.variables + ] + assert set(names) == set(SHARE_READERS) + for target in (simulation, simulation.baseline): + known_before.append( + { + name: list(target.get_holder(name).get_known_periods()) + for name in names + if target.get_holder(name).get_known_periods() + } + ) + return inject(simulation, shares, periods, weight_variable=weight_variable) + + # The shared EngineBlock injects (microcosm#931). + monkeypatch.setattr(engine_population, "inject_pool_shares", watching) + resolver = _block_resolver(frame, tmp_path) + assert known_before == [{}, {}] + shares = resolver.receipt()["engine_pool_shares"] + assert shares["applied"] is True and shares["refusals"] == [] + # A counterfactual that moves an allocation key is refused outright. + with pytest.raises(ValueError, match="pool-share correction reads"): + resolver.counterfactual_delta( + { + "kind": "input_substitution_counterfactual", + "zeroed_input": "corporate_wealth", + "output_variable": "income_tax", + }, + P, + ) + resolver.compute("household", "corporate_land_value") + resolver.close_engine_block() + reach = resolver.receipt()["engine_population_reach"] + assert reach["clean"] is True, reach["violations"] + + +class _ReadsAnIncomeDecile(UKMeasureResolver): + """A block whose measures reached a rank over its own households.""" + + def close_engine_block(self): + self.compute("household", "household_income_decile") + super().close_engine_block() + + +def test_a_block_that_reads_a_rank_is_not_exact_and_names_it(tmp_path): + frame = _pool(differ=True) + _problem, _rows_, _metrics, receipt = resolve_uk_full_national_problem( + frame, + TargetRegistry([], country="uk"), + period=P, + scratch_dir=tmp_path, + blocks=K, + resolver_factory=_ReadsAnIncomeDecile, + local_grains=(), + ) + representation = receipt["engine_population_representation"] + assert representation["exact"] is False + reach = representation["population_reach"] + assert reach["clean_in_every_block"] is False + assert reach["refused"] == ["household_income_decile"] + assert receipt["block_sensitivity"]["caveat"] is not None + + +def test_a_block_measured_in_a_rebalancing_year_is_refused(tmp_path): + """policyengine-uk draws which Universal Credit claimants are new by + benefit-unit position from 2026, so a block measured there is refused, + corrected or not: recorded when the block is built and by its verdict.""" + + resolver = _block_resolver(_pool(differ=True), tmp_path, year=2026) + shares = resolver.receipt()["engine_pool_shares"] + assert shares["periods"] == [str(D), "2026"] + assert len(shares["refusals"]) == 1 + assert "from 2026 on" in shares["refusals"][0] + resolver.close_engine_block() + receipt = resolver.receipt() + reach = receipt["engine_population_reach"] + assert reach["clean"] is False + assert {"kind": "window_period", "name": None, "period": "2026"} in reach["window"][ + "violations" + ] diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_weight_share_formula_inputs.py b/packages/microcosm-build/tests/engine/uk/test_uk_weight_share_formula_inputs.py index 9466c0cf59..672457a9f9 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_weight_share_formula_inputs.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_weight_share_formula_inputs.py @@ -12,6 +12,7 @@ import pytest from microcosm.build.uk_runtime.full_measure import UK_WEIGHT_SHARE_FORMULA_INPUTS +from microcosm.frame.adapters.policyengine_uk_population import UK_ENGINE_POPULATION #: The allocation-key variable each weight-share formula divides by, as the #: engine's formula reads it (``x * w / sum(x * w)``). @@ -22,6 +23,18 @@ } +def test_the_constant_is_the_declarations_keys_and_the_engines_adds(system): + """The frame columns the measure step sums for each block's pool-share + correction are the inputs the UK declaration names for the key the + formula divides by, which are the engine's own adds (microcosm#931).""" + + assert dict(UK_ENGINE_POPULATION.corrected) == ALLOCATION_KEYS + for formula, key in ALLOCATION_KEYS.items(): + declared = UK_ENGINE_POPULATION.key_adds[key] + assert UK_WEIGHT_SHARE_FORMULA_INPUTS[formula] == declared + assert tuple(system.variables[key].adds) == declared + + @pytest.fixture(scope="module") def system(): from policyengine_uk import CountryTaxBenefitSystem diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py index 9be6c42043..aa06f12c9b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py @@ -1,5 +1,7 @@ """Shared full-build measure evaluation retains legacy engine/RNG boundaries.""" +import copy +import math from types import SimpleNamespace import numpy as np @@ -11,10 +13,81 @@ clone_uk_dataset_with_ladder_geography, ) from microcosm.calibrate import TargetRegistry, TargetSpec +from microcosm.frame.adapters.engine_population import ( + MEASURE_PASS, + EngineBlock, + PoolShare, + PopulationVerdict, + Violation, +) +from microcosm.frame.adapters.policyengine_uk_population import UK_ENGINE_POPULATION from test_support.microcosm_build.uk_full_population_graph import source_frame from test_support.microcosm_build.uk_ladder_rowwise_clone import ( toy_ladder as toy_ladder, ) +from test_support.microcosm_frame.engine_population_fakes import ( + BLOCKS, + GROWTH, + KEY_A, + D, + P, + block_shares, + fake_block, + fake_declaration, + install_fake_engine, + numbers_are_years, +) + + +def _clean_block_receipt(shares, *, periods=("2024", "2025")): + """What a UKMeasureResolver records for an engine block whose correction + and guard all held (microcosm#931): the shares it applied, both checks, + one growth factor per key and the weights, siblings on the block's keys, + and a clean measure-pass verdict.""" + + corrections = {formula: share.factor() for formula, share in sorted(shares.items())} + keys = sorted({share.key_variable for share in shares.values()}) + verdict = PopulationVerdict( + pass_name="measure", + violations=(), + truncated=False, + classes={ + "corrected": tuple(sorted(UK_ENGINE_POPULATION.corrected)), + "stored": tuple(sorted(UK_ENGINE_POPULATION.stored)), + "zero_at_policy": ("carbon_tax",), + "inert": (), + }, + window_first_year=UK_ENGINE_POPULATION.window.first_year, + declaration_sha256=UK_ENGINE_POPULATION.sha256, + engine="policyengine-uk", + engine_version="test", + ) + return { + "engine_pool_shares": { + "periods": list(periods), + "dataset_year": int(periods[0]), + "formulas": {f: s.key_variable for f, s in sorted(shares.items())}, + "corrections": corrections, + "declaration_valid": True, + "key_sums_verified": True, + "largest_key_sum_gap": 1e-8, + "uniform_growth_verified": True, + "growth_factors": { + period: { + **{key: 1.03 for key in keys}, + UK_ENGINE_POPULATION.weight_variable: 1.004, + } + for period in periods[1:] + }, + "applied": True, + "siblings_match": True, + "refusals": [], + }, + "engine_population_reach": verdict.as_receipt(), + "annual_income_measures": { + "other_spells_engine_factors": {"employment_income_before_lsr": 1.04} + }, + } def test_full_measure_reuses_one_resolver( @@ -68,6 +141,8 @@ def receipt(self): ) assert len(constructions) == 1 + # One block over the pool is never corrected or judged (microcosm#931). + assert "engine_pool_shares" not in constructions[0] # The resolution loop's own receipt rides the measure receipt, one per # engine block; its shape is the resolver's, checked by its keys here. resolution = receipt.pop("resolution") @@ -135,6 +210,7 @@ def test_full_measure_resolves_real_per_clone_blocks( source_lineage_modulus=None, ) constructions = [] + closed = [] class StubResolver: def __init__(self, **kwargs): @@ -142,9 +218,15 @@ def __init__(self, **kwargs): self.simulation = object() self.contract_targets = {} self.cgt_period = 2024 if len(constructions) == 1 else second_cgt_period + self.shares = kwargs.get("engine_pool_shares") + + def close_engine_block(self): + closed.append(len(constructions)) def receipt(self): receipt = {"mode": "stub", "policyengine_uk_version": "test"} + if self.shares is not None: + receipt.update(_clean_block_receipt(self.shares)) if self.cgt_period is not None: receipt["cgt_period_contract"] = { "version": "uk-cgt-measurement-v2", @@ -186,15 +268,31 @@ def resolve(): prepared, restore, _, metrics, receipt = resolve() assert len(constructions) == 2 - # Each block carries half the pool, so its engine weights are scaled by two - # and the receipt records an exact (identical-copy) representation. + # Each block carries half the pool, so its engine weights are scaled by two. assert [call["engine_weight_scale"] for call in constructions] == pytest.approx( [2.0, 2.0] ) + # Each block's resolver gets the pool shares of its share formulas: the + # clones are identical copies, so c_b = f_b * S_b / S is 1, and each share + # carries the block's engine weight scale (microcosm#931). Every block is + # judged once it has computed everything. + for call in constructions: + shares = call["engine_pool_shares"] + assert set(shares) == set(UK_ENGINE_POPULATION.corrected) + for formula, share in shares.items(): + assert isinstance(share, PoolShare) + assert share.key_variable == UK_ENGINE_POPULATION.corrected[formula] + assert share.weight_scale == call["engine_weight_scale"] + assert share.factor() == pytest.approx(1.0, rel=1e-12) + assert closed == [1, 2] representation = receipt["engine_population_representation"] assert representation["mode"] == "block_weights_scaled_to_pool" assert representation["exact"] is True + assert representation["exact_basis"] == full_measure.UK_ENGINE_BLOCK_EXACT_BASIS + assert representation["exact_scope"] == full_measure.UK_ENGINE_BLOCK_EXACT_SCOPE assert representation["checks"]["identical_source_weight_multisets"] is True + assert representation["share_correction"]["applied_in_every_block"] is True + assert representation["population_reach"]["clean_in_every_block"] is True assert receipt["block_sensitivity"]["exact"] is True assert receipt["block_sensitivity"]["caveat"] is None assert [len(call["frame"].table("household")) for call in constructions] == [ @@ -210,14 +308,15 @@ def resolve(): } assert receipt["deviation"] == "per_clone_block_engine_resolution" sensitivity = receipt["block_sensitivity"] - assert ( - "ons/corporate_land_value" - in sensitivity["known_population_normalised_measures"] - ) - assert set(sensitivity["present_in_this_run"]) <= set( - sensitivity["known_population_normalised_measures"] - ) - # Identical clone copies scaled to the pool: the known formulas see the + # The engine variables the land rows bind, matched against the measure + # inputs (keyed by engine variable), not the rows' metric names. + assert sensitivity["known_population_normalised_measures"] == [ + "corporate_land_value", + "land_value", + ] + assert sensitivity["present_in_this_run"] == [] + assert sensitivity["corrected_formulas"] == dict(UK_ENGINE_POPULATION.corrected) + # Pool-scaled weights and corrected shares: the known formulas see the # pool's denominator, so the receipt names the mitigation and no caveat. assert "scaled by pool mass / block mass" in sensitivity["mitigation"] assert sensitivity["exact"] is True @@ -276,7 +375,7 @@ def receipt(self): ("household", "raw_engine_input"): provider.frame.table("household")[ "household_id" ].to_numpy(), - ("household", "ons/corporate_land_value"): np.ones( + ("household", "corporate_land_value"): np.ones( provider.frame.n("household") ), }, @@ -320,8 +419,11 @@ def materialize(adapter, _registry, **kwargs): assert metrics == {} assert receipt["national_inputs"] == 3 if blocks == 2: + # A measure input on the engine variable a land row binds is seen + # (microcosm#931: the record compared metric names with engine + # variable names, so it was always empty). assert receipt["block_sensitivity"]["present_in_this_run"] == [ - "ons/corporate_land_value" + "corporate_land_value" ] @@ -600,19 +702,27 @@ def _representation_block( ) -def test_engine_population_representation_is_exact_only_for_identical_copies(): +def test_engine_population_representation_records_the_blocks_before_they_run(): + """Before the blocks run, the representation records each block's factor + and allocation-key totals and the identical-copies checks, as diagnostics; + whether the blocks stand for the pool is decided after they ran, so exact + is provisional (microcosm#931).""" pool = SimpleNamespace(weights_for=lambda entity: SimpleNamespace(total=6.0)) copies = [ (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5])), (1, _representation_block([12, 10, 11], [1.5, 0.5, 1.0])), ] - exact = full_measure._engine_population_representation(pool, copies) - assert exact["exact"] is True - assert exact["factor_by_block"] == { + record = full_measure._engine_population_representation(pool, copies) + assert record["exact"] is False + assert record["exact_basis"] == "pool_share_injection_with_population_guard" + assert record["mode"] == "block_weights_scaled_to_pool" + assert record["factor_by_block"] == { "0": pytest.approx(2.0), "1": pytest.approx(2.0), } - assert exact["checks"]["max_abs_mass_share_deviation"] == pytest.approx(0.0) + assert record["checks"]["max_abs_mass_share_deviation"] == pytest.approx(0.0) + assert record["checks"]["identical_source_weight_multisets"] is True + assert record["checks"]["weight_share_inputs_match"] is True skewed = [ (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5])), @@ -620,50 +730,344 @@ def test_engine_population_representation_is_exact_only_for_identical_copies(): ] pool = SimpleNamespace(weights_for=lambda entity: SimpleNamespace(total=6.5)) approximate = full_measure._engine_population_representation(pool, skewed) - assert approximate["exact"] is False assert approximate["checks"]["identical_source_weight_multisets"] is False assert approximate["factor_by_block"]["0"] == pytest.approx(6.5 / 3.0) assert approximate["factor_by_block"]["1"] == pytest.approx(6.5 / 3.5) single = full_measure._engine_population_representation(pool, [(None, object())]) assert single == {"mode": "single_block", "exact": True, "blocks": 1} + assert full_measure._block_pool_shares(single) is None + assert full_measure._finalize_representation(single, [{}], shares_built=False) == { + "mode": "single_block", + "exact": True, + "blocks": 1, + } - # microcosm#1115 review: identical (source, weight) multisets are not - # enough; the allocation keys of the weight-share formulas must carry the - # same sum(x * w) in every block, and a frame without a source identity is - # never exact. pool = SimpleNamespace(weights_for=lambda entity: SimpleNamespace(total=6.0)) - keyed = [ - (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5], key_values=[1, 2, 3])), - (1, _representation_block([12, 10, 11], [1.5, 0.5, 1.0], key_values=[3, 1, 2])), - ] - assert full_measure._engine_population_representation(pool, keyed)["exact"] is True - drawn = [ - (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5], key_values=[1, 2, 3])), - (1, _representation_block([12, 10, 11], [1.5, 0.5, 1.0], key_values=[9, 1, 2])), - ] - inexact = full_measure._engine_population_representation(pool, drawn) - assert inexact["exact"] is False - assert inexact["checks"]["identical_source_weight_multisets"] is True - assert inexact["checks"]["weight_share_inputs_match"] is False - shares = inexact["checks"]["weight_share_inputs"]["shareholding"] - assert shares["match"] is False and shares["max_rel_diff"] > 0.1 - assert set(shares["sum_by_block"]) == {"0", "1"} nameless = [ (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5], identity=False)), (1, _representation_block([12, 10, 11], [1.5, 0.5, 1.0], identity=False)), ] anonymous = full_measure._engine_population_representation(pool, nameless) - assert anonymous["exact"] is False assert anonymous["checks"]["identity_key"] is None assert anonymous["checks"]["source_identity_present"] is False + + +def test_identical_copies_take_a_unit_correction_and_differing_keys_split_the_pool(): + pool = SimpleNamespace(weights_for=lambda entity: SimpleNamespace(total=6.0)) + copies = [ + (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5], key_values=[1, 2, 3])), + (1, _representation_block([12, 10, 11], [1.5, 0.5, 1.0], key_values=[3, 1, 2])), + ] + shares = full_measure._block_pool_shares( + full_measure._engine_population_representation(pool, copies) + ) + assert set(shares) == {"0", "1"} + for block in shares.values(): + assert set(block) == set(UK_ENGINE_POPULATION.corrected) + for formula, share in block.items(): + assert share.key_variable == UK_ENGINE_POPULATION.corrected[formula] + assert share.factor() == pytest.approx(1.0, rel=1e-12) + + # Clones that drew other keys: each block's c_b / f_b is its share S_b / S + # of the pool's total, and the shares add up to the whole. + drawn = [ + (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5], key_values=[1, 2, 3])), + (1, _representation_block([12, 10, 11], [1.5, 0.5, 1.0], key_values=[9, 1, 2])), + ] + representation = full_measure._engine_population_representation(pool, drawn) + assert representation["checks"]["weight_share_inputs_match"] is False + shares = full_measure._block_pool_shares(representation) + for formula in UK_ENGINE_POPULATION.corrected: + parts = [ + shares[block][formula].factor() / shares[block][formula].weight_scale + for block in ("0", "1") + ] + assert math.fsum(parts) == pytest.approx(1.0, rel=1e-12) + assert parts[0] != pytest.approx(parts[1]) + sums = representation["checks"]["weight_share_inputs"][formula]["sum_by_block"] + assert parts[1] == pytest.approx(sums["1"] / (sums["0"] + sums["1"])) + + +def test_a_block_without_a_key_column_gets_no_correction(monkeypatch, tmp_path): + pool = SimpleNamespace(weights_for=lambda entity: SimpleNamespace(total=6.0)) partial = [ (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5])), (1, _representation_block([12, 10, 11], [1.5, 0.5, 1.0])), ] partial[1][1].table("household").drop(columns=["corporate_wealth"], inplace=True) - unverified = full_measure._engine_population_representation(pool, partial) - assert unverified["exact"] is False - assert unverified["checks"]["weight_share_inputs"]["corporate_land_value"][ + representation = full_measure._engine_population_representation(pool, partial) + assert representation["checks"]["weight_share_inputs"]["corporate_land_value"][ "missing_columns" ] == ["corporate_wealth"] + assert full_measure._block_pool_shares(representation) is None + finalized = full_measure._finalize_representation( + representation, + [{"mode": "stub"}, {"mode": "stub"}], + shares_built=False, + ) + assert finalized["exact"] is False + assert finalized["share_correction"]["keys_complete"] is False + assert finalized["share_correction"]["applied_in_every_block"] is False + + +def _finalize_fixture(): + pool = SimpleNamespace(weights_for=lambda entity: SimpleNamespace(total=6.0)) + blocks = [ + (0, _representation_block([10, 11, 12], [0.5, 1.0, 1.5], key_values=[1, 2, 3])), + (1, _representation_block([12, 10, 11], [1.5, 0.5, 1.0], key_values=[9, 1, 2])), + ] + representation = full_measure._engine_population_representation(pool, blocks) + shares = full_measure._block_pool_shares(representation) + receipts = [_clean_block_receipt(shares[block]) for block in ("0", "1")] + return representation, receipts + + +def test_clean_corrected_blocks_are_exact_however_their_clones_differ(): + representation, receipts = _finalize_fixture() + finalized = full_measure._finalize_representation( + representation, receipts, shares_built=True + ) + assert finalized["exact"] is True + assert finalized["exact_basis"] == full_measure.UK_ENGINE_BLOCK_EXACT_BASIS + # The identical-copies checks fail and stay as diagnostics. + assert finalized["checks"]["weight_share_inputs_match"] is False + correction = finalized["share_correction"] + assert correction["formulas"] == dict(UK_ENGINE_POPULATION.corrected) + assert correction["periods"] == ["2024", "2025"] + assert correction["applied_in_every_block"] is True + assert correction["growth_factors_agree_across_blocks"] is True + assert correction["max_growth_factor_spread"] == 0.0 + assert correction["corrections_by_block"] == { + block: receipt["engine_pool_shares"]["corrections"] + for block, receipt in zip(("0", "1"), receipts, strict=True) + } + reach = finalized["population_reach"] + assert reach["pass"] == "measure" + assert reach["clean_in_every_block"] is True + assert reach["declaration_sha256"] == UK_ENGINE_POPULATION.sha256 + assert reach["classes"]["corrected"] == sorted(UK_ENGINE_POPULATION.corrected) + assert reach["refused"] == [] + assert reach["window"] == {"first_year": 2026, "violations": []} + assert finalized["annual_income_projection_factors"] == { + "agree_across_blocks": True, + "max_rel_spread": 0.0, + } + + +def _verdict(receipt, *violations): + receipt["engine_population_reach"]["clean"] = not violations + receipt["engine_population_reach"]["violations"] = [ + Violation(*violation).as_receipt() for violation in violations + ] + window = [v for v in violations if v[0].startswith("window_")] + receipt["engine_population_reach"]["window"]["violations"] = [ + Violation(*violation).as_receipt() for violation in window + ] + + +def _set(path, value): + def apply(receipt): + node = receipt + for part in path[:-1]: + node = node[part] + node[path[-1]] = value + + return apply + + +_FINALIZE_FAILURES = [ + pytest.param(change, id=reason) + for change, reason in ( + (_set(("engine_pool_shares", "applied"), False), "not applied"), + (_set(("engine_pool_shares", "key_sums_verified"), False), "check (a)"), + (_set(("engine_pool_shares", "uniform_growth_verified"), False), "check (b)"), + (_set(("engine_pool_shares", "siblings_match"), False), "siblings moved"), + (_set(("engine_pool_shares", "siblings_match"), None), "never closed"), + (_set(("engine_pool_shares", "declaration_valid"), False), "declaration"), + (_set(("engine_pool_shares", "refusals"), ["refused"]), "a refusal"), + (_set(("engine_pool_shares", "periods"), ["2024"]), "other periods"), + ( + _set(("engine_pool_shares", "growth_factors", "2025", "consumption"), 1.04), + "growth factors disagree", + ), + ( + _set( + ( + "annual_income_measures", + "other_spells_engine_factors", + "employment_income_before_lsr", + ), + 1.05, + ), + "annual factors disagree", + ), + ( + lambda receipt: _verdict( + receipt, ("refused_ran", "household_income_decile", "2025") + ), + "a refused rank ran", + ), + ( + lambda receipt: _verdict(receipt, ("window_period", None, "2026")), + "the window", + ), + ( + lambda receipt: _verdict( + receipt, ("stored_derived", "attends_private_school", "2025") + ), + "a stored input was derived", + ), + (lambda receipt: receipt.pop("engine_population_reach"), "no verdict"), + (lambda receipt: receipt.pop("engine_pool_shares"), "no correction"), + ) +] + + +@pytest.mark.parametrize("change", _FINALIZE_FAILURES) +def test_exact_follows_every_block_s_correction_and_verdict(change): + representation, receipts = _finalize_fixture() + clean = full_measure._finalize_representation( + representation, copy.deepcopy(receipts), shares_built=True + ) + assert clean["exact"] is True + change(receipts[1]) + finalized = full_measure._finalize_representation( + representation, receipts, shares_built=True + ) + assert finalized["exact"] is False + + +def test_a_refused_read_and_the_window_are_named_in_the_representation(): + representation, receipts = _finalize_fixture() + _verdict( + receipts[0], + ("refused_ran", "household_income_decile", "2025"), + ("window_period", None, "2026"), + ) + receipts[0]["engine_population_reach"]["refused"] = ["household_income_decile"] + finalized = full_measure._finalize_representation( + representation, receipts, shares_built=True + ) + reach = finalized["population_reach"] + assert finalized["exact"] is False + assert reach["clean_in_every_block"] is False + assert reach["refused"] == ["household_income_decile"] + assert reach["window"]["violations"] == [ + {"kind": "window_period", "name": None, "period": "2026"} + ] + assert list(reach["violations_by_block"]) == ["0"] + assert {violation["name"] for violation in reach["violations_by_block"]["0"]} == { + "household_income_decile", + None, + } + + +def test_a_missing_block_receipt_is_never_exact(): + representation, receipts = _finalize_fixture() + finalized = full_measure._finalize_representation( + representation, receipts[:1], shares_built=True + ) + assert finalized["exact"] is False + + +@pytest.fixture +def fake_engine(monkeypatch): + """The fake engine's own module, which its declaration names.""" + + return install_fake_engine(monkeypatch) + + +def _shared_block_receipt(block, failure=None): + """A resolver receipt carrying what the shared EngineBlock adds to it + (``receipt_entries``, as UKMeasureResolver.receipt does), from one block + of the fake engine's pool, with one of its checks made to fail.""" + + shares, _scales = block_shares(scaled=True) + own = shares[block] + simulation = fake_block(block) + rows = BLOCKS[block] + if failure == "check_a": + share = own["share"] + own = { + "share": PoolShare( + "share", + "key", + block_key_sum=share.block_key_sum * 1.01, + pool_key_sum=share.pool_key_sum, + weight_scale=share.weight_scale, + ) + } + elif failure == "check_b": + simulation.set_input("key_a", P, KEY_A[rows] * GROWTH * [1.0, 1.0, 1.1]) + engine_block = EngineBlock( + simulation, + own, + declaration=fake_declaration(), + policy=MEASURE_PASS, + periods=(D, P), + dataset_year=D, + ) + sibling = engine_block.wrap(lambda: fake_block(block))() + if failure == "siblings": + sibling.set_input("weight", D, np.ones(len(rows))) + engine_block.close() + return { + "mode": "stub", + "policyengine_uk_version": "test", + **engine_block.receipt_entries(), + } + + +def test_the_shared_blocks_receipts_are_what_the_finalisation_reads(fake_engine): + """Two blocks the shared EngineBlock corrected and judged clean, on the + fake engine, make an exact representation: the receipt the resolver + passes on is the one _finalize_representation reads (microcosm#931).""" + + representation, _receipts = _finalize_fixture() + receipts = [_shared_block_receipt(block) for block in ("0", "1")] + finalized = full_measure._finalize_representation( + representation, receipts, shares_built=True + ) + assert finalized["exact"] is True + correction = finalized["share_correction"] + assert correction["applied_in_every_block"] is True + assert correction["growth_factors_agree_across_blocks"] is True + assert correction["refusals_by_block"] == {} + assert finalized["population_reach"]["clean_in_every_block"] is True + + +@pytest.mark.parametrize( + "failure, flag", + [ + pytest.param("check_a", "key_sums_verified", id="check (a)"), + pytest.param("check_b", "uniform_growth_verified", id="check (b)"), + pytest.param("siblings", "siblings_match", id="a sibling moved"), + ], +) +def test_a_block_whose_check_fails_leaves_the_representation_inexact( + fake_engine, failure, flag +): + """A check the shared EngineBlock runs fails in one block: the block + records it, and the representation is not exact, naming the block and + holding only names and periods (microcosm#931).""" + + representation, _receipts = _finalize_fixture() + receipts = [_shared_block_receipt("0"), _shared_block_receipt("1", failure)] + shares = receipts[1]["engine_pool_shares"] + assert shares[flag] is False + finalized = full_measure._finalize_representation( + representation, receipts, shares_built=True + ) + assert finalized["exact"] is False + correction = finalized["share_correction"] + assert correction[flag] is False + refusals = correction["refusals_by_block"] + if failure == "siblings": + # Recorded by the flag alone: the correction was applied. + assert shares["applied"] is True and refusals == {} + else: + assert shares["applied"] is False + assert correction["applied_in_every_block"] is False + assert list(refusals) == ["1"] and len(refusals["1"]) == 1 + assert numbers_are_years(refusals["1"]) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py index cdcc5b011c..0dadb0b4d5 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py @@ -1963,6 +1963,175 @@ def __init__(self, *, dataset): ) +def _block_pool_shares(scale=4.0, **overrides): + from microcosm.frame.adapters.engine_population import PoolShare + from microcosm.frame.adapters.policyengine_uk_population import ( + UK_ENGINE_POPULATION, + ) + + shares = { + formula: PoolShare( + formula=formula, + key_variable=key, + block_key_sum=1.0e6, + pool_key_sum=4.2e6, + weight_scale=scale, + ) + for formula, key in UK_ENGINE_POPULATION.corrected.items() + } + shares.update(overrides) + return shares + + +def _fake_engine_resolver(monkeypatch, tmp_path, *, year=2025, **kwargs): + stub_scratch_engine_window(monkeypatch) + + class FakeMicrosimulation: + def __init__(self, *, dataset): + self.tax_benefit_system = SimpleNamespace(variables={}) + + monkeypatch.setitem( + sys.modules, + "policyengine_uk", + SimpleNamespace(__version__="9.9.9", Microsimulation=FakeMicrosimulation), + ) + monkeypatch.setattr( + measure_simulation, "write_uk_national_frame", lambda frame, path: path + ) + monkeypatch.setattr( + measure_simulation, "validate_uc_claimant_input", lambda *args, **kw: None + ) + return UKMeasureResolver( + simulation_source=None, + scratch_dir=tmp_path, + year=year, + frame=_tiny_national_frame([250.0, 125.0]), + **kwargs, + ) + + +def test_pool_shares_need_scratch_mode_and_one_blocks_shares(monkeypatch, tmp_path): + """An engine block's pool shares cover exactly the declared share formulas + over their keys, at the block's own weight scale, and need a scratch + export: a direct H5 is no block of a pool (microcosm#931).""" + from microcosm.frame.adapters.engine_population import PoolShare + + monkeypatch.setitem( + sys.modules, + "policyengine_uk", + SimpleNamespace(__version__="9.9.9", Microsimulation=lambda *, dataset: None), + ) + with pytest.raises(ValueError, match="pool shares need scratch-mode"): + UKMeasureResolver( + simulation_source=tmp_path / "input.h5", + scratch_dir=tmp_path, + year=2025, + frame=FrameStub(), + engine_pool_shares=_block_pool_shares(scale=1.0), + ) + shares = _block_pool_shares() + shares.pop("consumption_shareholding") + with pytest.raises(ValueError, match="must cover exactly"): + _fake_engine_resolver( + monkeypatch, tmp_path, engine_weight_scale=4.0, engine_pool_shares=shares + ) + with pytest.raises(ValueError, match="engine weight scale"): + _fake_engine_resolver( + monkeypatch, + tmp_path, + engine_weight_scale=2.0, + engine_pool_shares=_block_pool_shares(), + ) + misdeclared = _block_pool_shares( + shareholding=PoolShare( + formula="shareholding", + key_variable="consumption", + block_key_sum=1.0, + pool_key_sum=2.0, + weight_scale=4.0, + ) + ) + with pytest.raises(ValueError, match="not over corporate_sector_wealth"): + _fake_engine_resolver( + monkeypatch, + tmp_path, + engine_weight_scale=4.0, + engine_pool_shares=misdeclared, + ) + + +def test_an_engine_the_declaration_no_longer_matches_is_recorded_not_corrected( + monkeypatch, tmp_path +): + """The measure pass records (María's ruling of 2026-11-06): an engine the + UK declaration does not validate against leaves the block uncorrected and + unjudged, says why, and raises nothing.""" + resolver = _fake_engine_resolver( + monkeypatch, + tmp_path, + engine_weight_scale=4.0, + engine_pool_shares=_block_pool_shares(), + ) + resolver.close_engine_block() + receipt = resolver.receipt() + shares = receipt["engine_pool_shares"] + assert shares["declaration_valid"] is False + assert shares["applied"] is False + assert shares["key_sums_verified"] is False + assert shares["siblings_match"] is None + assert shares["periods"] == ["2024", "2025"] + assert shares["corrections"] == { + formula: pytest.approx(4.0 * 1.0e6 / 4.2e6) for formula in shares["formulas"] + } + assert len(shares["refusals"]) == 1 + assert "no longer matches the installed engine" in shares["refusals"][0] + assert "engine_population_reach" not in receipt + + +@pytest.mark.parametrize("year", [2025, 2026]) +def test_only_an_engine_block_records_its_correction(monkeypatch, tmp_path, year): + """A resolver that is no engine block (the national line, scoring, tools) + keeps its receipt's keys, and the position window does not apply to it, + in a rebalancing year too (microcosm#931).""" + resolver = _fake_engine_resolver( + monkeypatch, tmp_path, year=year, engine_weight_scale=4.0 + ) + resolver.close_engine_block() + receipt = resolver.receipt() + assert "engine_pool_shares" not in receipt + assert "engine_population_reach" not in receipt + resolver._note_engine_request("household_income_decile") + assert resolver._engine_pool_block is None + + +def test_an_engine_block_refuses_a_sibling_that_moves_what_its_correction_reads( + monkeypatch, tmp_path +): + resolver = _fake_engine_resolver( + monkeypatch, + tmp_path, + engine_weight_scale=4.0, + engine_pool_shares=_block_pool_shares(), + ) + for zeroed, folded in ( + ("corporate_wealth", None), + ("salary_sacrifice", "household_weight"), + ("transport_consumption", None), + ): + with pytest.raises(ValueError, match="pool-share correction reads"): + resolver._counterfactual_simulation(zeroed, folded) + monkeypatch.setattr( + measure_simulation, + "_ANNUAL_INCOME_HELD_VARIABLES", + (*measure_simulation._ANNUAL_INCOME_HELD_VARIABLES, "private_pension_wealth"), + ) + with pytest.raises(ValueError, match="annual-income simulation would set"): + resolver._annual_income_simulation() + # Outside an engine block nothing is refused for it. + plain = _fake_engine_resolver(monkeypatch, tmp_path, engine_weight_scale=4.0) + plain._refuse_pool_share_substitution(["corporate_wealth"], "a counterfactual") + + def test_the_other_spells_factor_is_the_engines_growth_parameters( monkeypatch, tmp_path ): diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py index 6bb32603b7..325b07482b 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py @@ -44,6 +44,13 @@ constructor applies is switched on. The verdict carries names, periods and booleans only. +One block's life. :class:`EngineBlock` runs these on a block's own engine, +in order: it validates the declaration, refuses work in the window, runs +checks (a) and (b), injects the shares and arms the recorder; it gives every +simulation the caller builds afterwards the same arrays and recorder, and at +the block's end checks that each still holds the block's keys and weights and +judges them all. A country's caller passes its declaration and a pass. + Batches. :func:`household_batches` splits a frame by a household column (whole households, groups nested in households) and :func:`scatter_by_entity_ids` puts each batch's results back in the order one @@ -69,7 +76,7 @@ import operator import types import weakref -from collections.abc import Iterable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, field from importlib import metadata from pathlib import Path @@ -85,6 +92,7 @@ "SPINE_BATCHED_PASS", "AllOf", "AnyOf", + "EngineBlock", "FormulaRun", "FormulaRunRecorder", "HouseholdBatch", @@ -117,6 +125,7 @@ "scatter_by_entity_ids", "source_sha256", "validate_declaration", + "validate_pool_shares", ] @@ -1068,6 +1077,20 @@ def watched(self) -> frozenset[str]: names |= {rule.sink, *rule.levels, *rule.carriers} return frozenset(names) + def correction_reads(self) -> frozenset[str]: + """What a pool-share correction reads: the corrected formulas, their + keys, the keys' inputs and the weight variable. No simulation a block + builds may set any of them (:meth:`EngineBlock.refuse_substitution`).""" + + return frozenset( + { + *self.corrected, + *self.corrected.values(), + *(add for adds in self.key_adds.values() for add in adds), + self.weight_variable, + } + ) + def run_predicates(self) -> dict[str, Predicate]: """The predicate checked at every run of each formula that has one.""" @@ -1839,6 +1862,322 @@ def _judge_inert( flag("inert_request", requested_name, period) +# --------------------------------------------------------------------------- +# One block's life +# --------------------------------------------------------------------------- + + +def validate_pool_shares( + declaration: PopulationDeclaration, + shares: Mapping[str, PoolShare], + *, + weight_scale: float | None = None, +) -> None: + """Refuse pool shares that are not one block's, for the declared formulas. + + One :class:`PoolShare` per formula the declaration corrects, over its + declared key. With ``weight_scale`` (the factor the block's engine + weights carry, 1 when they are unscaled) every share must carry it too. + A caller's error, so it raises under any pass. + """ + + expected = dict(declaration.corrected) + if set(shares) != set(expected): + raise ValueError( + f"engine pool shares must cover exactly {sorted(expected)}; got " + f"{sorted(shares)}." + ) + for formula, share in shares.items(): + if not isinstance(share, PoolShare): + raise TypeError(f"{formula}: engine pool shares must be PoolShare records.") + if share.formula != formula or share.key_variable != expected[formula]: + raise ValueError( + f"{formula}: the pool share is for {share.formula} over " + f"{share.key_variable}, not over {expected[formula]}." + ) + if weight_scale is not None and share.weight_scale != float(weight_scale): + raise ValueError( + f"{formula}: the pool share's weight factor is not the block's " + "engine weight scale." + ) + + +class EngineBlock: + """One engine block or batch standing for the pool, over its life. + + Built right after the block's simulation, before anything reads it, from + the block's :class:`PoolShare` records (:func:`validate_pool_shares`), + the declaration of the engine's whole-population formulas and the pass. + ``periods`` are the years the block computes at and ``dataset_year`` the + year its data is for. In order, on the block's own engine: + + - the declaration is validated against the simulation's tax-benefit + system (:func:`validate_declaration`), here and never on one engine + over the whole pool; + - a period in the declaration's position window is refused, as the + verdict will be: a block computing there is not the pool, corrected + or not; + - a period before the dataset's year is refused: the engine does not + project back to it, so a correction from that year's totals does not + hold there; + - check (a) at the dataset's year (:func:`check_key_sums`); + - check (b) from the dataset's year to every later period, on each key + and on the weight variable (:func:`check_uniform_growth`); + - every corrected formula is set at every period in the simulation and + its baseline (:func:`inject_pool_shares`); + - a :class:`FormulaRunRecorder` for the declaration is armed on both. + + Every simulation the caller builds afterwards takes the same arrays and + the recorder through :meth:`wrap`; one that would set what the + correction reads is refused by :meth:`refuse_substitution`. + :meth:`note_request` records what the caller asks for. :meth:`close`, at + the block's end, checks that every simulation still holds the block's + keys and weights, bitwise, judges them all under the pass and disarms. + Work after that is refused: nothing would judge it. + + Under a ``record`` pass a refusal leaves the block uncorrected or not + judged, and says why (:attr:`refusals`), so the caller's result is not + exact; under ``raise`` it raises. :meth:`receipt` holds names, periods, + booleans, the corrections and the growth factors: ratios of + block-weighted totals and parameter-driven factors, no record's value. + """ + + def __init__( + self, + simulation: Any, + shares: Mapping[str, PoolShare], + *, + declaration: PopulationDeclaration, + policy: PassPolicy, + periods: Iterable[int], + dataset_year: int, + ) -> None: + validate_pool_shares(declaration, shares) + self.main = simulation + self.shares = dict(shares) + self.declaration = declaration + self.policy = policy + self.dataset_year = int(dataset_year) + self.periods = tuple(sorted({int(period) for period in periods})) + self.declaration_valid = False + self.injected: InjectedShares | None = None + self.recorder: FormulaRunRecorder | None = None + self.key_sum_gaps: dict[str, float] = {} + self.key_sums_verified = False + self.growth_factors: dict[str, dict[str, float | None]] = {} + self.uniform_growth_verified = False + self.siblings: list[Any] = [] + self.siblings_match: bool | None = None + self.requested: set[tuple[str, int]] = set() + self.refusals: list[str] = [] + self.verdict: PopulationVerdict | None = None + self.closed = False + try: + validate_declaration(declaration, simulation.tax_benefit_system) + except PopulationDeclarationError as error: + self._refuse(error) + return + self.declaration_valid = True + window = declaration.window + inside = [period for period in self.periods if window.contains(period)] + if inside: + self._refuse( + PoolShareError( + f"the block computes at {inside}, from {window.first_year} " + f"on, where {window.reason}." + ) + ) + try: + self._correct(simulation) + except PoolShareError as error: + self._refuse(error) + self.recorder = FormulaRunRecorder.for_declaration(declaration) + self.recorder.arm(simulation) + + def _refuse(self, error: Exception) -> None: + if self.policy.on_violation == "raise": + raise error + self.refusals.append(str(error)) + + def _correct(self, simulation: Any) -> None: + weight = self.declaration.weight_variable + early = [period for period in self.periods if period < self.dataset_year] + if early: + raise PoolShareError( + f"the block computes at {early}, before the dataset's year " + f"{self.dataset_year}, which the engine does not project to, so " + "a correction from that year's totals does not hold there." + ) + self.key_sum_gaps = check_key_sums( + simulation, self.shares, self.dataset_year, weight_variable=weight + ) + self.key_sums_verified = True + carried = (*sorted(set(self.declaration.corrected.values())), weight) + for period in self.periods: + if period == self.dataset_year: + continue + factors = check_uniform_growth( + simulation, carried, self.dataset_year, period + ) + for name, factor in factors.items(): + if factor is not None and not (math.isfinite(factor) and factor > 0): + raise PoolShareError( + f"The engine does not carry {name} from " + f"{self.dataset_year} to {period}." + ) + self.growth_factors[str(period)] = factors + self.uniform_growth_verified = True + self.injected = inject_pool_shares( + simulation, self.shares, self.periods, weight_variable=weight + ) + + def wrap(self, factory: Callable[..., Any]) -> Callable[..., Any]: + """``factory``, giving each simulation it builds the block's arrays + and recorder before the caller sets any input on it.""" + + def build(*args: Any, **kwargs: Any) -> Any: + sibling = factory(*args, **kwargs) + if self.closed: + # Not armed: arming would record again, and nothing judges it. + self._refuse( + PoolShareError( + "A simulation was built after the block was judged, " + "so nothing judges what it computes." + ) + ) + return sibling + if self.injected is not None: + try: + self.injected.apply(sibling) + except PoolShareError as error: + self._refuse(error) + if self.recorder is not None: + self.recorder.arm(sibling) + self.siblings.append(sibling) + return sibling + + return build + + def note_request(self, name: str, period: int) -> None: + """Record that the caller asked for ``name`` at ``period``, which + the verdict judges (a refused formula, the window).""" + + if self.closed: + self._refuse( + PoolShareError( + f"{name} was asked for at {int(period)} after the block was " + "judged, so nothing judges it." + ) + ) + return + self.requested.add((str(name), int(period))) + + def refuse_substitution(self, names: Iterable[str | None], route: str) -> None: + """Refuse a simulation that would set what the block's correction + reads (:meth:`PopulationDeclaration.correction_reads`): its shares + would no longer be the pool's. Raises under any pass, before the + simulation is built.""" + + reads = self.declaration.correction_reads() + touched = sorted({str(name) for name in names if name} & reads) + if touched: + raise ValueError( + f"{route} would set {touched}, which the block's pool-share " + "correction reads (the share formulas, their allocation keys " + "and the keys' inputs, and the weights), so its shares would no " + "longer be the pool's." + ) + + def close(self) -> PopulationVerdict | None: + """Judge the block's simulations once, at its end, and disarm. + + Returns the verdict, None when the declaration did not validate (so + nothing was armed). Under a ``raise`` pass a simulation that no + longer holds the block's keys or weights raises + :class:`PoolShareError`, and a verdict that is not clean + :class:`PopulationGuardError`. + """ + + if self.closed: + return self.verdict + self.closed = True + if self.recorder is None: + return None + simulations = [self.main, *self.siblings] + try: + if self.injected is not None: + self.siblings_match = all( + self.injected.matches(simulation) for simulation in simulations + ) + if not self.siblings_match: + self._refuse_moved_keys() + self.verdict = judge( + self.declaration, + self.policy, + self.recorder, + simulations, + self.periods, + sorted(self.requested), + ) + finally: + self.recorder.disarm() + return self.verdict + + def _refuse_moved_keys(self) -> None: + # Recorded by siblings_match under a record pass. + if self.policy.on_violation == "raise": + raise PoolShareError( + "A simulation the block built no longer holds the block's keys " + "or weights, so its shares are not the pool's." + ) + + def _corrections(self) -> dict[str, float | None]: + corrections: dict[str, float | None] = {} + for formula, share in sorted(self.shares.items()): + try: + corrections[formula] = share.factor() + except PoolShareError: + corrections[formula] = None + return corrections + + def receipt(self) -> dict[str, Any]: + """The correction: names, periods, booleans, ratios of block-weighted + totals and parameter-driven growth factors.""" + + return { + "periods": [str(period) for period in self.periods], + "dataset_year": self.dataset_year, + "formulas": { + formula: share.key_variable + for formula, share in sorted(self.shares.items()) + }, + "corrections": self._corrections(), + "declaration_valid": self.declaration_valid, + "key_sums_verified": self.key_sums_verified, + "largest_key_sum_gap": ( + max(self.key_sum_gaps.values()) if self.key_sum_gaps else None + ), + "uniform_growth_verified": self.uniform_growth_verified, + "growth_factors": { + period: dict(sorted(factors.items())) + for period, factors in sorted(self.growth_factors.items()) + }, + "applied": self.injected is not None, + "siblings_match": self.siblings_match, + "refusals": list(self.refusals), + } + + def receipt_entries(self) -> dict[str, Any]: + """What the block adds to its caller's receipt: ``engine_pool_shares`` + (:meth:`receipt`) and, once judged, ``engine_population_reach`` (the + verdict).""" + + entries: dict[str, Any] = {"engine_pool_shares": self.receipt()} + if self.verdict is not None: + entries["engine_population_reach"] = self.verdict.as_receipt() + return entries + + # --------------------------------------------------------------------------- # Batches # --------------------------------------------------------------------------- diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py b/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py index 72fc319103..4983a29151 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py @@ -3,12 +3,13 @@ A fake simulation stands in for a policyengine-core one: holders that know which periods hold inputs, formulas run through ``_run_formula``, a parameter tree read at a period, and a ``clone`` that rebinds the methods -bound to the original, as core's does. Everything is synthetic. +bound to the original, as core's does. The fake engine lives in +``test_support.microcosm_frame.engine_population_fakes``, which the build +tests share. Everything is synthetic. """ from __future__ import annotations -import copy import math import sys import types @@ -23,13 +24,13 @@ FLOAT32_UNIT_ROUNDOFF, MEASURE_PASS, SPINE_BATCHED_PASS, + EngineBlock, FormulaRunRecorder, InertRule, ParameterTest, PassPolicy, PoolShare, PoolShareError, - PopulationDeclaration, PopulationDeclarationError, PopulationGuardError, PositionWindow, @@ -50,352 +51,52 @@ scatter_by_entity_ids, source_sha256, validate_declaration, + validate_pool_shares, +) +from test_support.microcosm_frame.engine_population_fakes import ( + BLOCKS, + FAKE_ENGINE, + GROWTH, + KEY_A, + WEIGHT, + D, + FakeSimulation, + N, + P, + install_fake_engine, + numbers_are_years, +) +from test_support.microcosm_frame.engine_population_fakes import ( + block_shares as _block_shares, +) +from test_support.microcosm_frame.engine_population_fakes import ( + fake_block as _block, +) +from test_support.microcosm_frame.engine_population_fakes import ( + fake_declaration as _declaration, +) +from test_support.microcosm_frame.engine_population_fakes import ( + fake_inputs as _inputs, +) +from test_support.microcosm_frame.engine_population_fakes import ( + fake_pool as _pool, +) +from test_support.microcosm_frame.engine_population_fakes import ( + fake_system as _fake_system, +) +from test_support.microcosm_frame.engine_population_fakes import ( + frame_sum as _frame_sum, +) +from test_support.microcosm_frame.engine_population_fakes import ( + local_formula as _local, ) - -D, P = 2024, 2025 -N = 6 -FAKE_ENGINE = "fake_population_engine" - - -# --------------------------------------------------------------------------- -# A fake engine -# --------------------------------------------------------------------------- - - -def _namespace(tree): - if isinstance(tree, dict): - return SimpleNamespace( - **{key: _namespace(value) for key, value in tree.items()} - ) - return tree - - -class FakeParameters: - """A parameter tree: called with a period it gives that period's values, - and its root's children are reachable as attributes, as core's are.""" - - def __init__(self, tree, overrides=None): - self.tree = tree - self.overrides = overrides or {} - - def __call__(self, period): - tree = copy.deepcopy(self.tree) - for path, value in self.overrides.get(int(str(period)[:4]), {}).items(): - *parents, leaf = path.split(".") - node = tree - for parent in parents: - node = node[parent] - node[leaf] = value - return _namespace(tree) - - def __getattr__(self, name): - tree = self.__dict__["tree"] - if name in tree: - return _namespace(tree[name]) - raise AttributeError(name) - - -class FakeVariable: - def __init__(self, name, *, formula=None, adds=None, dtype=np.float32): - self.name = name - self.entity = SimpleNamespace(key="household") - self.formula = formula - self.formulas = {"0000-01-01": formula} if formula is not None else {} - self.adds = adds - self.subtracts = None - self.dtype = dtype - - -class FakeHolder: - def __init__(self, variable): - self.variable = variable - self.arrays: dict[str, np.ndarray] = {} - self.inputs: set[str] = set() - - def get_known_periods(self): - return list(self.arrays) - - def get_input_periods(self, branch_name="default"): - return [period for period in self.arrays if period in self.inputs] - - def get_array(self, period, branch_name="default"): - return self.arrays.get(str(period)) - - def copy(self): - holder = FakeHolder(self.variable) - holder.arrays = {key: value.copy() for key, value in self.arrays.items()} - holder.inputs = set(self.inputs) - return holder - - -class FakeSimulation: - def __init__(self, system, inputs, *, rows=N, baseline=True): - self.tax_benefit_system = system - self.rows = rows - self.branch_name = "default" - self.branches = {} - self.holders = { - name: FakeHolder(variable) for name, variable in system.variables.items() - } - for (name, period), values in inputs.items(): - self.set_input(name, period, values) - if baseline: - self.baseline = self.clone() - - def get_holder(self, name): - return self.holders[name] - - def set_input(self, name, period, values): - holder = self.holders[name] - holder.arrays[str(period)] = np.array(values, dtype=holder.variable.dtype) - holder.inputs.add(str(period)) - - def calculate(self, name, period): - holder = self.holders[name] - cached = holder.arrays.get(str(period)) - if cached is not None: - return cached - variable = holder.variable - result = self._run_formula(variable, None, period) - if result is None: - values = np.zeros(self.rows, dtype=variable.dtype) - else: - values = np.broadcast_to( - np.asarray(result).astype(variable.dtype), (self.rows,) - ).copy() - holder.arrays[str(period)] = values - return values - - def _run_formula(self, variable, population, period): - if variable.formula is not None: - return variable.formula(self, period) - if variable.adds: - total = 0 - for name in variable.adds: - total = total + self.calculate(name, period) - return total - return None - - def clone(self): - """Copy as policyengine-core does: every attribute but the branches, - with the methods bound to the original bound to the copy.""" - - new = object.__new__(type(self)) - for key, value in self.__dict__.items(): - if key != "branches": - new.__dict__[key] = value - for key, value in list(new.__dict__.items()): - if isinstance(value, types.MethodType) and value.__self__ is self: - new.__dict__[key] = types.MethodType(value.__func__, new) - new.holders = {name: holder.copy() for name, holder in self.holders.items()} - new.branches = {} - return new - - def get_branch(self, name): - if name not in self.branches: - branch = self.clone() - branch.branch_name = name - branch.parent_branch = self - self.branches[name] = branch - return self.branches[name] - - -def _share(simulation, period): - key = simulation.calculate("key", period).astype(np.float64) - weight = simulation.calculate("weight", period).astype(np.float64) - total = float(np.sum(key * weight)) - return key / total if total else np.zeros_like(key) - - -def _rates(simulation, period): - return simulation.calculate("share", period) * 1_000.0 - - -def _budget_change(simulation, period): - budget = simulation.tax_benefit_system.parameters(period).gov.budget - if budget == 0: - return 0 - weight = simulation.calculate("weight", period) - return budget / weight.sum() + 0 * weight - - -def _months(simulation, period): - return np.full(simulation.rows, 6.0) - - -def _attendance(simulation, period): - income = simulation.calculate("local", period) - return income > np.median(income) - - -def _vat(rate_path): - def formula(simulation, period): - node = simulation.tax_benefit_system.parameters(period) - for part in rate_path.split("."): - node = getattr(node, part) - return simulation.calculate("attendance", period).astype(np.float64) * node - - return formula - - -def _vat_change(simulation, period): - return simulation.calculate("vat", period) - simulation.calculate( - "baseline_vat", period - ) - - -def _tax_total(simulation, period): - return simulation.calculate("vat", period) + simulation.calculate("rates", period) - - -def _decile(simulation, period): - income = simulation.calculate("local", period) - return np.argsort(np.argsort(income)) % 10 - - -def _local(simulation, period): - return simulation.calculate("key_a", period) * 2.0 - - -def _fake_system(*, budget=0.0, vat_rate=0.2, baseline_vat_rate=0.2): - names = { - "weight": FakeVariable("weight"), - "key_a": FakeVariable("key_a"), - "key_b": FakeVariable("key_b"), - "key": FakeVariable("key", adds=["key_a", "key_b"]), - "share": FakeVariable("share", formula=_share), - "rates": FakeVariable("rates", formula=_rates), - "months": FakeVariable("months", formula=_months), - "attendance": FakeVariable("attendance", formula=_attendance, dtype=bool), - "vat": FakeVariable("vat", formula=_vat("gov.vat_rate")), - "baseline_vat": FakeVariable( - "baseline_vat", formula=_vat("baseline.gov.vat_rate") - ), - "vat_change": FakeVariable("vat_change", formula=_vat_change), - "tax_total": FakeVariable("tax_total", formula=_tax_total), - "budget_change": FakeVariable("budget_change", formula=_budget_change), - "decile": FakeVariable("decile", formula=_decile, dtype=np.int64), - "local": FakeVariable("local", formula=_local), - } - tree = { - "gov": {"budget": budget, "vat_rate": vat_rate}, - "baseline": {"gov": {"vat_rate": baseline_vat_rate}}, - } - return SimpleNamespace(variables=names, parameters=FakeParameters(tree)) - - -def _quiet_reform(parameters, period): - return None - - -def _loud_reform(parameters, period): - return object() - - -def _late_reform(parameters, period): - return object() if int(str(period)[:4]) == P else None @pytest.fixture def fake_engine(monkeypatch): """The engine's own module: its list, its window and its reform factories.""" - module = types.ModuleType(FAKE_ENGINE) - module.WHOLE_POPULATION = frozenset( - {"share", "months", "attendance", "budget_change", "decile"} - ) - module.WINDOW = range(2026, 2033) - module.quiet_reform = _quiet_reform - module.loud_reform = _loud_reform - module.late_reform = _late_reform - monkeypatch.setitem(sys.modules, FAKE_ENGINE, module) - return module - - -def _declaration(**changes): - fields = { - "engine": FAKE_ENGINE, - "distribution": "fake-population-engine", - "reviewed_version": "1.0.0", - "list_source": f"{FAKE_ENGINE}:WHOLE_POPULATION", - "weight_variable": "weight", - "corrected": {"share": "key"}, - "key_adds": {"key": ("key_a", "key_b")}, - "stored": ("months", "attendance"), - "zero_at_policy": {"budget_change": ParameterTest("gov.budget", "eq", 0)}, - "inert": { - "attendance": InertRule( - sink="vat_change", - levels=("vat", "baseline_vat"), - carriers=("tax_total",), - predicate=all_of( - ParameterTest("gov.vat_rate", "eq_path", "baseline.gov.vat_rate") - ), - ) - }, - "refused": ("decile",), - "window": PositionWindow(2026, f"{FAKE_ENGINE}:WINDOW", "positional draws"), - "observed": ("rates",), - "constructor_factories": (f"{FAKE_ENGINE}:quiet_reform",), - } - fields.update(changes) - return PopulationDeclaration(**fields) - - -# Two clones of three households each: the first block's keys are larger. -KEY_A = np.array([10.0, 0.0, 30.0, 1.0, 2.0, 3.0]) -KEY_B = np.array([5.0, 5.0, 0.0, 1.0, 0.0, 2.0]) -WEIGHT = np.array([100.0, 200.0, 300.0, 150.0, 150.0, 150.0]) -BLOCKS = {"0": np.array([0, 1, 2]), "1": np.array([3, 4, 5])} -GROWTH = 1.03 - - -def _inputs(rows, *, weight_scale=1.0, store=("months", "attendance")): - inputs = {} - for period, factor in ((D, 1.0), (P, GROWTH)): - inputs[("key_a", period)] = KEY_A[rows] * factor - inputs[("key_b", period)] = KEY_B[rows] * factor - inputs[("weight", period)] = WEIGHT[rows] * weight_scale - if "months" in store: - inputs[("months", period)] = np.full(len(rows), 3.0) - if "attendance" in store: - inputs[("attendance", period)] = np.zeros(len(rows), dtype=bool) - return inputs - - -def _frame_sum(rows): - return float(np.sum((KEY_A[rows] + KEY_B[rows]) * WEIGHT[rows])) - - -def _block_shares(*, scaled): - pool_mass = float(WEIGHT.sum()) - scales = ( - {block: pool_mass / float(WEIGHT[rows].sum()) for block, rows in BLOCKS.items()} - if scaled - else None - ) - shares = pool_shares( - {block: {"share": _frame_sum(rows)} for block, rows in BLOCKS.items()}, - scales, - key_variables={"share": "key"}, - ) - return shares, scales - - -def _block(block, *, scaled=True, store=("months", "attendance"), system=None): - rows = BLOCKS[block] - scale = 1.0 - if scaled: - scale = float(WEIGHT.sum()) / float(WEIGHT[rows].sum()) - return FakeSimulation( - system or _fake_system(), - _inputs(rows, weight_scale=scale, store=store), - rows=len(rows), - ) - - -def _pool(): - return FakeSimulation(_fake_system(), _inputs(np.arange(N))) + return install_fake_engine(monkeypatch) # --------------------------------------------------------------------------- @@ -1314,6 +1015,280 @@ def leaves(value): assert not isinstance(leaf, float) +# --------------------------------------------------------------------------- +# One block's life +# --------------------------------------------------------------------------- + + +def _open_block( + block="0", + *, + simulation=None, + shares=None, + policy=MEASURE_PASS, + periods=(D, P), +): + scaled, _scales = _block_shares(scaled=True) + return EngineBlock( + _block(block) if simulation is None else simulation, + scaled[block] if shares is None else shares, + declaration=_declaration(), + policy=policy, + periods=periods, + dataset_year=D, + ) + + +def _off_frame_total(block="0"): + """The block's share with a key total 1% off the engine's.""" + + shares, _scales = _block_shares(scaled=True) + share = shares[block]["share"] + return { + "share": PoolShare( + "share", + "key", + block_key_sum=share.block_key_sum * 1.01, + pool_key_sum=share.pool_key_sum, + weight_scale=share.weight_scale, + ) + } + + +def test_an_engine_block_corrects_wraps_and_judges_what_it_builds(fake_engine): + pool = _pool() + block = _open_block() + receipt = block.receipt() + assert receipt["declaration_valid"] is True + assert receipt["key_sums_verified"] is True + assert receipt["uniform_growth_verified"] is True + assert receipt["applied"] is True + assert receipt["refusals"] == [] + assert receipt["periods"] == [str(D), str(P)] + assert receipt["growth_factors"] == { + str(P): {"key": pytest.approx(GROWTH, rel=1e-6), "weight": 1.0} + } + assert receipt["siblings_match"] is None + sibling = block.wrap(lambda: _block("0"))() + for simulation in (block.main, sibling, sibling.baseline): + np.testing.assert_allclose( + simulation.calculate("share", P), + pool.calculate("share", P)[BLOCKS["0"]], + rtol=1e-6, + ) + block.note_request("tax_total", P) + sibling.calculate("tax_total", P) + verdict = block.close() + assert verdict is block.verdict and verdict.clean, verdict.describe() + assert block.receipt()["siblings_match"] is True + assert block.receipt_entries() == { + "engine_pool_shares": block.receipt(), + "engine_population_reach": verdict.as_receipt(), + } + # Disarmed, and judged once. + assert "_run_formula" not in vars(block.main) + assert "_run_formula" not in vars(sibling) + assert block.close() is verdict + + +def test_a_block_whose_engine_key_total_is_not_the_frames_stays_uncorrected( + fake_engine, +): + """Check (a) fails: recorded, the shares are never set, and the verdict + then finds the share formula no input where the block computes.""" + + block = _open_block(shares=_off_frame_total()) + receipt = block.receipt() + assert receipt["declaration_valid"] is True + assert receipt["key_sums_verified"] is False + assert receipt["uniform_growth_verified"] is False + assert receipt["applied"] is False + (refusal,) = receipt["refusals"] + assert "frame total" in refusal + assert numbers_are_years(receipt["refusals"]) + verdict = block.close() + assert ("corrected_not_input", "share", str(D)) in _kinds(verdict) + assert block.receipt()["siblings_match"] is None + + +def _grows_by_row(inputs): + # The third row's key grows by 1.0 where the others grow by 1.03. + inputs[("key_a", P)] = inputs[("key_a", D)] * np.array([GROWTH, GROWTH, 1.0]) + + +def _weights_grow_by_row(inputs): + inputs[("weight", P)] = inputs[("weight", D)] * np.array([1.0, 1.0, 1.1]) + + +def _a_zero_row_turns_nonzero(inputs): + # The second row carries no key in the dataset's year, and does later. + inputs[("key_b", D)] = np.array([5.0, 0.0, 0.0]) + inputs[("key_b", P)] = np.array([5.0 * GROWTH, 4.0, 0.0]) + + +@pytest.mark.parametrize( + "change, message", + [ + pytest.param(_grows_by_row, "one factor per row", id="a key by row"), + pytest.param(_weights_grow_by_row, "one factor per row", id="weights by row"), + pytest.param(_a_zero_row_turns_nonzero, "zero at", id="a zero row"), + ], +) +def test_a_block_its_engine_does_not_carry_by_one_factor_stays_uncorrected( + fake_engine, change, message +): + """Check (b) fails: the dataset-year correction would not hold later.""" + + inputs = _inputs(BLOCKS["0"]) + change(inputs) + simulation = FakeSimulation(_fake_system(), inputs, rows=3) + # The shares are the block's own at the dataset's year, so check (a) + # holds and check (b) is what refuses. + total = float( + np.sum((inputs[("key_a", D)] + inputs[("key_b", D)]) * inputs[("weight", D)]) + ) + shares = { + "share": PoolShare("share", "key", block_key_sum=total, pool_key_sum=3 * total) + } + block = _open_block(simulation=simulation, shares=shares) + receipt = block.receipt() + assert receipt["key_sums_verified"] is True + assert receipt["uniform_growth_verified"] is False + assert receipt["growth_factors"] == {} + assert receipt["applied"] is False + (refusal,) = receipt["refusals"] + assert message in refusal + assert numbers_are_years(receipt["refusals"]) + + +@pytest.mark.parametrize( + "move", + [ + pytest.param(lambda s: s.set_input("weight", D, np.ones(3)), id="weights"), + pytest.param(lambda s: s.set_input("key", P, np.zeros(3)), id="a key"), + pytest.param( + lambda s: s.baseline.set_input("weight", P, np.ones(3)), + id="its baseline's weights", + ), + ], +) +def test_a_sibling_that_moves_the_keys_or_weights_once_built_fails_the_block( + fake_engine, move +): + block = _open_block() + sibling = block.wrap(lambda: _block("0"))() + move(sibling) + block.close() + assert block.receipt()["siblings_match"] is False + assert block.receipt()["applied"] is True + + +def test_a_block_computing_in_the_window_or_before_its_data_is_refused(fake_engine): + late = _open_block(periods=(D, 2026)) + assert "from 2026 on" in late.refusals[0] + assert ("window_period", None, "2026") in _kinds(late.close()) + early = _open_block(periods=(2023, D)) + receipt = early.receipt() + assert receipt["applied"] is False and receipt["key_sums_verified"] is False + (refusal,) = receipt["refusals"] + assert "before the dataset's year" in refusal + assert numbers_are_years([*late.refusals, *early.refusals]) + + +def test_an_engine_its_declaration_does_not_describe_is_recorded_never_armed(): + # Without the fake engine's module the declaration's own list cannot be + # read, so it does not validate. + block = _open_block() + receipt = block.receipt() + assert receipt["declaration_valid"] is False + assert receipt["applied"] is False + (refusal,) = receipt["refusals"] + assert "no longer matches the installed engine" in refusal + assert block.recorder is None and "_run_formula" not in vars(block.main) + assert block.close() is None and block.verdict is None + assert set(block.receipt_entries()) == {"engine_pool_shares"} + + +def test_a_raising_pass_raises_what_the_measure_pass_records(fake_engine): + with pytest.raises(PoolShareError, match="frame total"): + _open_block(shares=_off_frame_total(), policy=SPINE_BATCHED_PASS) + with pytest.raises(PoolShareError, match="from 2026 on"): + _open_block(periods=(D, 2026), policy=SPINE_BATCHED_PASS) + block = _open_block(policy=SPINE_BATCHED_PASS) + block.main.calculate("decile", P) + with pytest.raises(PopulationGuardError, match="decile"): + block.close() + assert "_run_formula" not in vars(block.main) + block = _open_block(policy=SPINE_BATCHED_PASS) + sibling = block.wrap(lambda: _block("0"))() + sibling.set_input("weight", D, np.ones(3)) + with pytest.raises(PoolShareError, match="no longer holds"): + block.close() + assert "_run_formula" not in vars(sibling) + + +def test_a_raising_pass_raises_on_an_engine_its_declaration_does_not_describe(): + with pytest.raises(PopulationDeclarationError): + _open_block(policy=SPINE_BATCHED_PASS) + + +def test_a_block_refuses_a_simulation_that_sets_what_its_correction_reads( + fake_engine, +): + assert _declaration().correction_reads() == { + "share", + "key", + "key_a", + "key_b", + "weight", + } + block = _open_block() + for names in (["key_a"], [None, "weight"], ["share"], ["key"]): + with pytest.raises(ValueError, match="pool-share correction reads"): + block.refuse_substitution(names, "a counterfactual") + block.refuse_substitution(["local", None], "a counterfactual") + + +def test_work_after_the_block_is_judged_is_refused_and_never_armed(fake_engine): + block = _open_block() + build = block.wrap(lambda: _block("0")) + verdict = block.close() + assert verdict.clean + late = build() + block.note_request("tax_total", P) + assert len(block.refusals) == 2 + assert all("after the block was judged" in text for text in block.refusals) + assert numbers_are_years(block.refusals) + assert "_run_formula" not in vars(late) + assert block.siblings == [] and block.requested == set() + assert block.close() is verdict + + +def test_pool_shares_are_one_blocks_for_the_declared_formulas(): + shares, scales = _block_shares(scaled=True) + declaration = _declaration() + validate_pool_shares(declaration, shares["0"]) + validate_pool_shares(declaration, shares["0"], weight_scale=scales["0"]) + with pytest.raises(ValueError, match="engine weight scale"): + validate_pool_shares(declaration, shares["0"], weight_scale=1.0) + with pytest.raises(ValueError, match="must cover exactly"): + validate_pool_shares(declaration, {}) + with pytest.raises(TypeError, match="PoolShare records"): + validate_pool_shares(declaration, {"share": 1.0}) + over_weights = PoolShare("share", "weight", block_key_sum=1.0, pool_key_sum=2.0) + with pytest.raises(ValueError, match="not over key"): + validate_pool_shares(declaration, {"share": over_weights}) + with pytest.raises(ValueError, match="must cover exactly"): + EngineBlock( + _block("0"), + {}, + declaration=declaration, + policy=MEASURE_PASS, + periods=(D,), + dataset_year=D, + ) + + # --------------------------------------------------------------------------- # The declaration # --------------------------------------------------------------------------- diff --git a/test_support/microcosm_frame/engine_population_fakes.py b/test_support/microcosm_frame/engine_population_fakes.py new file mode 100644 index 0000000000..ef5d1a35b0 --- /dev/null +++ b/test_support/microcosm_frame/engine_population_fakes.py @@ -0,0 +1,386 @@ +"""A fake policyengine-core engine for the shared engine-block machinery. + +A fake simulation stands in for a policyengine-core one: holders that know +which periods hold inputs, formulas run through ``_run_formula``, a +parameter tree read at a period, and a ``clone`` that rebinds the methods +bound to the original, as core's does. Its declaration names a fake engine +module, which :func:`install_fake_engine` puts in ``sys.modules``. Two clones +of three households each make the blocks. The shared tests and the build +tests that hold a block's receipt to the measure step's finalisation use it +(microcosm#931). Everything is synthetic. +""" + +from __future__ import annotations + +import copy +import re +import sys +import types +from types import SimpleNamespace + +import numpy as np + +from microcosm.frame.adapters.engine_population import ( + InertRule, + ParameterTest, + PopulationDeclaration, + PositionWindow, + all_of, + pool_shares, +) + +D, P = 2024, 2025 +N = 6 +FAKE_ENGINE = "fake_population_engine" + + +# --------------------------------------------------------------------------- +# A fake engine +# --------------------------------------------------------------------------- + + +def _namespace(tree): + if isinstance(tree, dict): + return SimpleNamespace( + **{key: _namespace(value) for key, value in tree.items()} + ) + return tree + + +class FakeParameters: + """A parameter tree: called with a period it gives that period's values, + and its root's children are reachable as attributes, as core's are.""" + + def __init__(self, tree, overrides=None): + self.tree = tree + self.overrides = overrides or {} + + def __call__(self, period): + tree = copy.deepcopy(self.tree) + for path, value in self.overrides.get(int(str(period)[:4]), {}).items(): + *parents, leaf = path.split(".") + node = tree + for parent in parents: + node = node[parent] + node[leaf] = value + return _namespace(tree) + + def __getattr__(self, name): + tree = self.__dict__["tree"] + if name in tree: + return _namespace(tree[name]) + raise AttributeError(name) + + +class FakeVariable: + def __init__(self, name, *, formula=None, adds=None, dtype=np.float32): + self.name = name + self.entity = SimpleNamespace(key="household") + self.formula = formula + self.formulas = {"0000-01-01": formula} if formula is not None else {} + self.adds = adds + self.subtracts = None + self.dtype = dtype + + +class FakeHolder: + def __init__(self, variable): + self.variable = variable + self.arrays: dict[str, np.ndarray] = {} + self.inputs: set[str] = set() + + def get_known_periods(self): + return list(self.arrays) + + def get_input_periods(self, branch_name="default"): + return [period for period in self.arrays if period in self.inputs] + + def get_array(self, period, branch_name="default"): + return self.arrays.get(str(period)) + + def copy(self): + holder = FakeHolder(self.variable) + holder.arrays = {key: value.copy() for key, value in self.arrays.items()} + holder.inputs = set(self.inputs) + return holder + + +class FakeSimulation: + def __init__(self, system, inputs, *, rows=N, baseline=True): + self.tax_benefit_system = system + self.rows = rows + self.branch_name = "default" + self.branches = {} + self.holders = { + name: FakeHolder(variable) for name, variable in system.variables.items() + } + for (name, period), values in inputs.items(): + self.set_input(name, period, values) + if baseline: + self.baseline = self.clone() + + def get_holder(self, name): + return self.holders[name] + + def set_input(self, name, period, values): + holder = self.holders[name] + holder.arrays[str(period)] = np.array(values, dtype=holder.variable.dtype) + holder.inputs.add(str(period)) + + def calculate(self, name, period): + holder = self.holders[name] + cached = holder.arrays.get(str(period)) + if cached is not None: + return cached + variable = holder.variable + result = self._run_formula(variable, None, period) + if result is None: + values = np.zeros(self.rows, dtype=variable.dtype) + else: + values = np.broadcast_to( + np.asarray(result).astype(variable.dtype), (self.rows,) + ).copy() + holder.arrays[str(period)] = values + return values + + def _run_formula(self, variable, population, period): + if variable.formula is not None: + return variable.formula(self, period) + if variable.adds: + total = 0 + for name in variable.adds: + total = total + self.calculate(name, period) + return total + return None + + def clone(self): + """Copy as policyengine-core does: every attribute but the branches, + with the methods bound to the original bound to the copy.""" + + new = object.__new__(type(self)) + for key, value in self.__dict__.items(): + if key != "branches": + new.__dict__[key] = value + for key, value in list(new.__dict__.items()): + if isinstance(value, types.MethodType) and value.__self__ is self: + new.__dict__[key] = types.MethodType(value.__func__, new) + new.holders = {name: holder.copy() for name, holder in self.holders.items()} + new.branches = {} + return new + + def get_branch(self, name): + if name not in self.branches: + branch = self.clone() + branch.branch_name = name + branch.parent_branch = self + self.branches[name] = branch + return self.branches[name] + + +def _share(simulation, period): + key = simulation.calculate("key", period).astype(np.float64) + weight = simulation.calculate("weight", period).astype(np.float64) + total = float(np.sum(key * weight)) + return key / total if total else np.zeros_like(key) + + +def _rates(simulation, period): + return simulation.calculate("share", period) * 1_000.0 + + +def _budget_change(simulation, period): + budget = simulation.tax_benefit_system.parameters(period).gov.budget + if budget == 0: + return 0 + weight = simulation.calculate("weight", period) + return budget / weight.sum() + 0 * weight + + +def _months(simulation, period): + return np.full(simulation.rows, 6.0) + + +def _attendance(simulation, period): + income = simulation.calculate("local", period) + return income > np.median(income) + + +def _vat(rate_path): + def formula(simulation, period): + node = simulation.tax_benefit_system.parameters(period) + for part in rate_path.split("."): + node = getattr(node, part) + return simulation.calculate("attendance", period).astype(np.float64) * node + + return formula + + +def _vat_change(simulation, period): + return simulation.calculate("vat", period) - simulation.calculate( + "baseline_vat", period + ) + + +def _tax_total(simulation, period): + return simulation.calculate("vat", period) + simulation.calculate("rates", period) + + +def _decile(simulation, period): + income = simulation.calculate("local", period) + return np.argsort(np.argsort(income)) % 10 + + +def local_formula(simulation, period): + return simulation.calculate("key_a", period) * 2.0 + + +def fake_system(*, budget=0.0, vat_rate=0.2, baseline_vat_rate=0.2): + names = { + "weight": FakeVariable("weight"), + "key_a": FakeVariable("key_a"), + "key_b": FakeVariable("key_b"), + "key": FakeVariable("key", adds=["key_a", "key_b"]), + "share": FakeVariable("share", formula=_share), + "rates": FakeVariable("rates", formula=_rates), + "months": FakeVariable("months", formula=_months), + "attendance": FakeVariable("attendance", formula=_attendance, dtype=bool), + "vat": FakeVariable("vat", formula=_vat("gov.vat_rate")), + "baseline_vat": FakeVariable( + "baseline_vat", formula=_vat("baseline.gov.vat_rate") + ), + "vat_change": FakeVariable("vat_change", formula=_vat_change), + "tax_total": FakeVariable("tax_total", formula=_tax_total), + "budget_change": FakeVariable("budget_change", formula=_budget_change), + "decile": FakeVariable("decile", formula=_decile, dtype=np.int64), + "local": FakeVariable("local", formula=local_formula), + } + tree = { + "gov": {"budget": budget, "vat_rate": vat_rate}, + "baseline": {"gov": {"vat_rate": baseline_vat_rate}}, + } + return SimpleNamespace(variables=names, parameters=FakeParameters(tree)) + + +def _quiet_reform(parameters, period): + return None + + +def _loud_reform(parameters, period): + return object() + + +def _late_reform(parameters, period): + return object() if int(str(period)[:4]) == P else None + + +def install_fake_engine(monkeypatch): + """Install the fake engine's own module, as the declaration names it: its + list, its window and its reform factories. Returns the module.""" + + module = types.ModuleType(FAKE_ENGINE) + module.WHOLE_POPULATION = frozenset( + {"share", "months", "attendance", "budget_change", "decile"} + ) + module.WINDOW = range(2026, 2033) + module.quiet_reform = _quiet_reform + module.loud_reform = _loud_reform + module.late_reform = _late_reform + monkeypatch.setitem(sys.modules, FAKE_ENGINE, module) + return module + + +def fake_declaration(**changes): + fields = { + "engine": FAKE_ENGINE, + "distribution": "fake-population-engine", + "reviewed_version": "1.0.0", + "list_source": f"{FAKE_ENGINE}:WHOLE_POPULATION", + "weight_variable": "weight", + "corrected": {"share": "key"}, + "key_adds": {"key": ("key_a", "key_b")}, + "stored": ("months", "attendance"), + "zero_at_policy": {"budget_change": ParameterTest("gov.budget", "eq", 0)}, + "inert": { + "attendance": InertRule( + sink="vat_change", + levels=("vat", "baseline_vat"), + carriers=("tax_total",), + predicate=all_of( + ParameterTest("gov.vat_rate", "eq_path", "baseline.gov.vat_rate") + ), + ) + }, + "refused": ("decile",), + "window": PositionWindow(2026, f"{FAKE_ENGINE}:WINDOW", "positional draws"), + "observed": ("rates",), + "constructor_factories": (f"{FAKE_ENGINE}:quiet_reform",), + } + fields.update(changes) + return PopulationDeclaration(**fields) + + +# Two clones of three households each: the first block's keys are larger. +KEY_A = np.array([10.0, 0.0, 30.0, 1.0, 2.0, 3.0]) +KEY_B = np.array([5.0, 5.0, 0.0, 1.0, 0.0, 2.0]) +WEIGHT = np.array([100.0, 200.0, 300.0, 150.0, 150.0, 150.0]) +BLOCKS = {"0": np.array([0, 1, 2]), "1": np.array([3, 4, 5])} +GROWTH = 1.03 + + +def fake_inputs(rows, *, weight_scale=1.0, store=("months", "attendance")): + inputs = {} + for period, factor in ((D, 1.0), (P, GROWTH)): + inputs[("key_a", period)] = KEY_A[rows] * factor + inputs[("key_b", period)] = KEY_B[rows] * factor + inputs[("weight", period)] = WEIGHT[rows] * weight_scale + if "months" in store: + inputs[("months", period)] = np.full(len(rows), 3.0) + if "attendance" in store: + inputs[("attendance", period)] = np.zeros(len(rows), dtype=bool) + return inputs + + +def frame_sum(rows): + return float(np.sum((KEY_A[rows] + KEY_B[rows]) * WEIGHT[rows])) + + +def block_shares(*, scaled): + pool_mass = float(WEIGHT.sum()) + scales = ( + {block: pool_mass / float(WEIGHT[rows].sum()) for block, rows in BLOCKS.items()} + if scaled + else None + ) + shares = pool_shares( + {block: {"share": frame_sum(rows)} for block, rows in BLOCKS.items()}, + scales, + key_variables={"share": "key"}, + ) + return shares, scales + + +def fake_block(block, *, scaled=True, store=("months", "attendance"), system=None): + rows = BLOCKS[block] + scale = 1.0 + if scaled: + scale = float(WEIGHT.sum()) / float(WEIGHT[rows].sum()) + return FakeSimulation( + system or fake_system(), + fake_inputs(rows, weight_scale=scale, store=store), + rows=len(rows), + ) + + +def fake_pool(): + return FakeSimulation(fake_system(), fake_inputs(np.arange(N))) + + +def numbers_are_years(texts) -> bool: + """Whether every number written in ``texts`` is a year, as a refusal's + must be: it names formulas, variables and periods, never a value or a + count.""" + + numbers = [ + number for text in texts for number in re.findall(r"\d+(?:\.\d+)?", text) + ] + return all(re.fullmatch(r"(19|20)\d\d", number) for number in numbers) From 935f706c3b134095ff60ab120627d38e05b9d935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:27:21 +0000 Subject: [PATCH 004/108] Batch spine engine calls by clone with the pool's share denominator (#931) One simulation over the move's K=15 spine pool does not fit in memory, so the UK adapter runs a call over more than one clone clone by clone. It decides from the frame, never from a constructor flag: a household table carrying household_clone_index with more than one clone batches, and a frame without the column, or with one clone, takes the single simulation it always has. National never batches. - The shared engine_population module gains materialize_in_batches and frame_key_sums. Before any engine is built, the driver judges the call's requests: a period in the rebalancing window, a refused rank, a stored formula the frame does not supply, and attendance or its levels and carriers while it is unstored. It forms each batch's share from the frame, c_b = S_b / S at unscaled weights. Each batch, in ascending clone order, then runs an EngineBlock under the spine's raising pass: checks (a) and (b), the pool's shares set in the simulation and its baseline, the recorder armed, the requested values, the verdict. The engine is deleted and collected before the next is built. Every batch must grow the keys and the weights by one factor. People come back in the frame's order and groups by id, as one engine returns them. The driver refuses a recording pass, since its arrays carry no verdict. - PolicyEngineUKEngine passes the UK declaration and the spine pass. It gains simulation_batches(frame), parameters(period) for the stage-52 restatement, and batch_record(). Its optional clones, the spine's K once G1 adds it, refuses a frame carrying any other number of clones, so no call runs per clone or over a sub-pool. microcosm.frame.rules and the RulesEngine.materialize signature are unchanged. - The spine sidecar's rules_engine gains engine_batches only once a call batched. The engine identity is still hashed from the provenance alone, and no batch cap is added. - graph_kernels binds engine_population and policyengine_uk_population beside the adapter in all 16 engine-calling stages. A test derives those stages from their engine calls. No data changes on today's lines: no spine frame carries household_clone_index before G1, so every call takes the single simulation, unchanged. Identities move. Every spine stage kernel hashes graph_kernels.py, so every spine node re-keys, and the 16 engine stages also bind the two modules. engine_population.py is a target compile module, so the full and national target kernels re-key again. The spine graph declaration (the H2 fixture uk_spine.json), source_stages.json and the coverage manifest are unchanged. An engine test runs a synthetic three-clone frame whose clones differ, interleave by id and list people out of order, at the dataset year and projected a year on. Against one simulation over the same frame, the share formulas match within the float32 bound, household_tax within its per-row bound, and household-local values bitwise, with people and groups in one engine's order. A one-clone frame is bitwise the single simulation. A batched attendance request, a decile, a 2026 period, unstored months and a frame with fewer clones than K are each refused before any simulation is built. Engine-free tests run the driver on fake engines, one engine alive at a time. Co-Authored-By: Claude Opus 5.5 --- .../build/uk_runtime/graph_kernels.py | 47 +- .../microcosm/build/uk_runtime/spine_build.py | 22 +- .../tests/engine_free/uk/test_uk_graph.py | 99 ++++- .../uk/test_uk_rules_engine_provenance.py | 77 ++++ .../frame/adapters/engine_population.py | 300 ++++++++++++- .../frame/adapters/policyengine_uk.py | 161 ++++++- .../engine/uk/test_policyengine_uk_batches.py | 417 ++++++++++++++++++ .../shared/test_engine_population.py | 275 ++++++++++++ .../uk/test_policyengine_uk_adapter.py | 217 +++++++++ 9 files changed, 1574 insertions(+), 41 deletions(-) create mode 100644 packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_batches.py diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index c656e6fc37..f74ee877bc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -30,7 +30,11 @@ from microcosm.fit import donors as fit_donors from microcosm.fit import held_out as fit_held_out from microcosm.frame import Frame, MassChangeRecord, WeightKind +from microcosm.frame.adapters import engine_population from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter +from microcosm.frame.adapters import ( + policyengine_uk_population as uk_engine_population_declaration, +) from microcosm.graph import ( Capabilities, Determinism, @@ -153,6 +157,17 @@ "age_tail": "age_tail", } +# What every stage that calls the engine computes through: the adapter, which +# runs a call over more than one clone clone by clone, and the shared +# engine-block machinery with the UK declaration it judges each batch against +# (microcosm#931). An edit to any of them moves the hash of every such stage; +# test_uk_graph derives the stages from their engine calls. +_ENGINE_MODULES = ( + uk_engine_adapter, + engine_population, + uk_engine_population_declaration, +) + # Imported modules are not traversed by ``source_hash``. Bind relationship # helpers and the adapter's input-retention checks into every consuming stage. _STAGE_HELPER_MODULES = { @@ -171,14 +186,14 @@ # and its twin transport in spi_rank_transport (microcosm#840). "hmrc_spi_income_spine": (spi_income, spi_rank_transport), "frs_spine": (uc_relationships,), - "frs_legacy_proxies": (uk_engine_adapter,), - "frs_education_grant_split": (uk_engine_adapter,), - "frs_brma": (uk_engine_adapter,), - "was_wealth": (uk_engine_adapter,), + "frs_legacy_proxies": (*_ENGINE_MODULES,), + "frs_education_grant_split": (*_ENGINE_MODULES,), + "frs_brma": (*_ENGINE_MODULES,), + "was_wealth": (*_ENGINE_MODULES,), # The LISA stage reads its household donor through the WAS cleaning and its # household predictors through the WAS recipient surface; the anchor counts # HMRC's payers through the typed ISA facts (microcosm#1415). - "was_lisa": (uk_engine_adapter, was_wealth, hmrc_isa_statistics), + "was_lisa": (*_ENGINE_MODULES, was_wealth, hmrc_isa_statistics), # Landlords' receipts read the vendored PRIS rows through their typed # loader, the tape through the income stage's preparation and uprating, # and the support channel's names (microcosm#1106). @@ -189,14 +204,14 @@ # The purchases read the vendored HMRC Stamp Tax rows through the feed-pinned # reader and mark SDLT's regions from the FRS region map (microcosm#1095). "property_purchases": (ledger_fact_vendoring, rowwise_geography), - "nts_bus_travel": (uk_engine_adapter, bus_use_incidence), + "nts_bus_travel": (*_ENGINE_MODULES, bus_use_incidence), # The energy level and prices are energy_pricing's (microcosm#1418): a # code-only change to them must re-key the stage (the independent review # of microcosm#1420, finding 5). - "lcfs_consumption": (uk_engine_adapter, energy_pricing), - "etb_vat": (uk_engine_adapter,), - "etb_services": (uk_engine_adapter,), - "uc_reporter_redraw": (uc_relationships, uk_engine_adapter), + "lcfs_consumption": (*_ENGINE_MODULES, energy_pricing), + "etb_vat": (*_ENGINE_MODULES,), + "etb_services": (*_ENGINE_MODULES,), + "uc_reporter_redraw": (uc_relationships, *_ENGINE_MODULES), # The SPI benefit pass restores from the FRS twin through the support # channel's lineage, re-derives the disability flags and the own-right # rule, draws over the take-up population and maps household weights to @@ -218,7 +233,7 @@ # attributes, the take-up population and the family-type roles, and counts # its pools and receipt on the source lineage (microcosm#1334). "uc_take_up": ( - uk_engine_adapter, + *_ENGINE_MODULES, ledger_fact_vendoring, legacy_closure_take_up, uc_deduction_attributes, @@ -232,20 +247,20 @@ "uc_legacy_closure_take_up": (uc_capital_coherence, ledger_fact_vendoring), # The Pension Credit redraw reuses the UC stage's household-to-benefit-unit # weight mapping and the adapter's engine materialization. - "pension_credit_take_up": (uc_capital_coherence, uk_engine_adapter), + "pension_credit_take_up": (uc_capital_coherence, *_ENGINE_MODULES), # The Child Benefit redraw shares the weight mapping and the adapter. - "child_benefit_take_up": (uc_capital_coherence, uk_engine_adapter), + "child_benefit_take_up": (uc_capital_coherence, *_ENGINE_MODULES), # The Carer's Allowance draw reads the earnings test through the adapter's # engine materialization and counts its receipt's support on the source # lineage (microcosm#1319), and balances its draws through the shared # placement helper (microcosm#1418). "carers_allowance_take_up": ( - uk_engine_adapter, + *_ENGINE_MODULES, receipt_counts, stochastic_assignment, ), # The stored attendance is one engine materialization (microcosm#1095). - "private_school_attendance": (uk_engine_adapter,), + "private_school_attendance": (*_ENGINE_MODULES,), # The subscriptions walk the CGT bands on its income proxy (with the # proxy's loss relief), read HMRC's ISA tables through their typed loader # and round their receipts' support (microcosm#1415). @@ -275,7 +290,7 @@ spi_income, spi_support, stochastic_assignment, - uk_engine_adapter, + *_ENGINE_MODULES, ), # The copies' childcare re-draw draws through frs_take_up's childcare # helper, reads each household's clone path, and measures its flags for diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index c967ed0585..726d8699bc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -755,9 +755,27 @@ def _rules_engine() -> PolicyEngineUKEngine: ) assert_installed_engine_is_pinned() + # Whether a stage's call runs clone by clone is decided from its frame, + # never set here (microcosm#931), so nothing about batching enters the + # engine identity hashed from _rules_engine_provenance. return PolicyEngineUKEngine() +def _rules_engine_record(engine: object) -> dict[str, object]: + """The sidecar's ``rules_engine``: the engine's provenance and, only once + one of its calls ran clone by clone, how they batched + (``engine_batches``, the adapter's :meth:`batch_record`, for the calls + this process ran). The engine identity is hashed from the provenance + alone, so it does not move with batching.""" + + record = _rules_engine_provenance() + batch_record = getattr(engine, "batch_record", None) + batches = batch_record() if callable(batch_record) else None + if batches is not None: + record["engine_batches"] = dict(batches) + return record + + def _rules_engine_provenance() -> dict[str, object]: try: version = metadata.version("policyengine-uk") @@ -915,6 +933,7 @@ def _build_sidecar( synthetic_fixture: Mapping[str, object] | None = None, staging_delivery: Mapping[str, object] | None = None, spine_gate_report: dict[str, object] | None = None, + engine: object | None = None, ) -> dict[str, object]: household_weight = frame.weights_for("household") return { @@ -958,7 +977,7 @@ def _build_sidecar( "staging_delivery": dict(staging_delivery or {}), "spine_gate_report": spine_gate_report, "stochastic_contract_sha256": stochastic_contract_sha256, - "rules_engine": _rules_engine_provenance(), + "rules_engine": _rules_engine_record(engine), # Which stages read the SPI Public Use Tape, whose Special Conditions # reach every output built on them (microcosm#1095). UK_SPI_PUBLIC_USE_TAPE_KEY: uk_spi_public_use_tape_marker(stages), @@ -2449,6 +2468,7 @@ def _main(argv: list[str] | None = None) -> int: if spine_gate_path.is_file() else None ), + engine=prepared.engine, ) sidecar["operation_inventory"] = list(uk_spine_operation_inventory(graph, spec)) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 9444996d62..1db745f68b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -335,45 +335,104 @@ def changed_helper_bytes(path: Path) -> bytes: assert registry.implementation_hash(control) == before[control] -def test_uk_adapter_source_changes_invalidate_all_consuming_stages(monkeypatch): - from microcosm.frame.adapters import policyengine_uk +def _calls_the_engine(module) -> bool: + """Whether a module calls a rules engine: ``engine.materialize`` or + ``self.engine.materialize``, or the predictor materialization of + ``microcosm.frame.rules``, which calls it.""" + import ast + import inspect - graph = uk_spine_graph() - registry = uk_registry(graph=graph) - stages = ( + tree = ast.parse(Path(inspect.getfile(module)).read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + function = node.func + if isinstance(function, ast.Attribute) and function.attr == "materialize": + target = function.value + if (isinstance(target, ast.Name) and target.id == "engine") or ( + isinstance(target, ast.Attribute) and target.attr == "engine" + ): + return True + name = getattr(function, "id", None) or getattr(function, "attr", None) + if name == "materialize_rules_engine_predictors": + return True + return False + + +def test_every_stage_that_calls_the_engine_binds_the_engine_modules() -> None: + """How an engine call runs, and clone by clone once a frame carries more + than one clone, is the adapter's, the shared engine-block machinery's and + the UK declaration's (microcosm#931). ``source_hash`` does not follow + imports, so every stage whose module calls the engine binds all three, + and no other stage binds any: one that only checks the engine's country + (isa_subscriptions) computes nothing through it.""" + from microcosm.build.uk_runtime import graph_kernels + + engine_modules = set(graph_kernels._ENGINE_MODULES) + calling = { + stage + for stage in graph_kernels._STAGE_MODULES + if _calls_the_engine(graph_kernels._stage_module(stage)) + } + assert { "frs_legacy_proxies", - "frs_education_grant_split", "frs_brma", - "was_wealth", "was_lisa", - "nts_bus_travel", - "lcfs_consumption", - "etb_vat", - "etb_services", - "uc_reporter_redraw", + "uc_take_up", + "private_school_attendance", + "company_cars", + } <= calling + assert "isa_subscriptions" not in calling + for stage in graph_kernels._STAGE_MODULES: + bound = set(graph_kernels._STAGE_HELPER_MODULES.get(stage, ())) + if stage in calling: + assert engine_modules <= bound, stage + else: + assert not engine_modules & bound, stage + + +@pytest.mark.parametrize( + "module_name", + ["policyengine_uk", "engine_population", "policyengine_uk_population"], +) +def test_engine_module_changes_invalidate_every_stage_that_calls_the_engine( + monkeypatch, module_name +): + import importlib + + from microcosm.build.uk_runtime import graph_kernels + + module = importlib.import_module(f"microcosm.frame.adapters.{module_name}") + graph = uk_spine_graph() + registry = uk_registry(graph=graph) + stages = sorted( + stage + for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() + if module in helpers ) + assert "frs_legacy_proxies" in stages and "private_school_attendance" in stages affected = [graph.node(stage).kernel for stage in stages] controls = [ graph.node(stage).kernel - for stage in ("frs_council_tax", "uc_capital_coherence") + for stage in ("frs_council_tax", "uc_capital_coherence", "isa_subscriptions") ] before = {ref: registry.implementation_hash(ref) for ref in affected + controls} - adapter_path = Path(policyengine_uk.__file__).resolve() + module_path = Path(module.__file__).resolve() original = Path.read_bytes - def changed_adapter_bytes(path): + def changed_module_bytes(path): content = original(path) return ( - content + b"\n# adapter-only change\n" - if path.resolve() == adapter_path + content + b"\n# engine-module-only change\n" + if path.resolve() == module_path else content ) - monkeypatch.setattr(Path, "read_bytes", changed_adapter_bytes) + monkeypatch.setattr(Path, "read_bytes", changed_module_bytes) for ref in affected: - assert registry.implementation_hash(ref) != before[ref] + assert registry.implementation_hash(ref) != before[ref], ref for ref in controls: - assert registry.implementation_hash(ref) == before[ref] + assert registry.implementation_hash(ref) == before[ref], ref def test_uk_graph_json_round_trip_is_canonical() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py index 7f97a7a1b8..b1f7da5d7c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py @@ -155,3 +155,80 @@ def missing(package: str) -> str: "package": "policyengine-uk", "version": "unavailable", } + + +class _BatchingEngine: + def __init__(self, record: dict | None) -> None: + self.record = record + + def batch_record(self) -> dict | None: + return self.record + + +_BATCHES = { + "by": "household_clone_index", + "batched_calls": 3, + "batches": [15], + "pass": "spine_batched", + "declaration_sha256": "ab" * 32, +} + + +def _sidecar(engine: object | None) -> dict: + import pandas as pd + + from microcosm.build.uk_runtime.national_frame import uk_national_frame + from microcosm.frame import WeightKind + + frame = uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 2], + "age": [30.0, 40.0], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + {"household_id": [1, 2], "household_weight": [1.0, 2.0]} + ), + time_period=2023, + weight_kind=WeightKind.DESIGN, + ) + return spine_build._build_sidecar( + frame=frame, + stages=[], + records=[], + artifact_pins={}, + resource_pins={}, + input_artifact_pins={}, + hmrc_replay={}, + stochastic_contract_sha256="0" * 64, + frs_vintage="synthetic", + sampling=None, + engine=engine, + ) + + +def test_the_sidecar_records_batches_only_once_a_call_batched( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # National never batches, and today no spine frame carries the clone + # index, so the sidecar's rules engine stays the provenance; a spine whose + # calls ran clone by clone adds how (microcosm#931). + _install(monkeypatch, None) + provenance = spine_build._rules_engine_provenance() + for engine in (None, object(), _BatchingEngine(None)): + assert spine_build._rules_engine_record(engine) == provenance + assert _sidecar(engine)["rules_engine"] == provenance + batched = _BatchingEngine(dict(_BATCHES)) + assert spine_build._rules_engine_record(batched) == { + **provenance, + "engine_batches": _BATCHES, + } + assert _sidecar(batched)["rules_engine"]["engine_batches"] == _BATCHES + # The engine identity is hashed from the provenance alone. + assert _identity(spine_build._rules_engine_provenance()) == _identity( + {"package": "policyengine-uk", "version": VERSION} + ) diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py index 325b07482b..53b824bab0 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py @@ -54,7 +54,13 @@ Batches. :func:`household_batches` splits a frame by a household column (whole households, groups nested in households) and :func:`scatter_by_entity_ids` puts each batch's results back in the order one -engine over the frame returns them. +engine over the frame returns them. :func:`materialize_in_batches` runs one +call that way: it judges the call's requests before any engine is built, +forms each batch's share from the frame's own totals +(:func:`frame_key_sums`, ``c_b = S_b / S`` at unscaled weights), runs an +:class:`EngineBlock` on each batch's engine under a pass that raises, holds +the batches to one growth factor per key, and releases each engine before the +next is built. Simulations are duck-typed policyengine-core simulations (``calculate``, ``set_input``, ``get_holder``, ``baseline``, ``branches``, ``branch_name``, @@ -67,6 +73,7 @@ from __future__ import annotations import functools +import gc import hashlib import importlib import importlib.util @@ -87,6 +94,7 @@ import pandas as pd __all__ = [ + "BATCH_FACTOR_AGREEMENT", "FLOAT32_UNIT_ROUNDOFF", "MEASURE_PASS", "SPINE_BATCHED_PASS", @@ -116,9 +124,11 @@ "derived_watched_periods", "factor_spread", "float32_sum_roundings", + "frame_key_sums", "household_batches", "inject_pool_shares", "judge", + "materialize_in_batches", "pool_share_atol", "pool_share_rtol", "pool_shares", @@ -2300,3 +2310,291 @@ def scatter_by_entity_ids( result = np.empty(len(target), dtype=values.dtype) result[slots] = values return result + + +# --------------------------------------------------------------------------- +# One call in batches +# --------------------------------------------------------------------------- + +#: How far each allocation key's and the weights' growth factors may spread +#: across the batches of one call, relative to their mean. They are the +#: engine's parameter-driven growth, equal in every batch to float32 +#: rounding, far inside this. +BATCH_FACTOR_AGREEMENT = 1e-6 + + +def frame_key_sums( + bundle: Any, + declaration: PopulationDeclaration, + batches: Sequence[HouseholdBatch], + *, + household: str = "household", +) -> dict[Any, dict[str, float]]: + """Each batch's ``sum(x * w)`` for every formula the declaration corrects. + + ``x`` is the formula's allocation key, the sum of its declared inputs + (``declaration.key_adds``) as the frame's ``household`` table holds + them, and ``w`` the frame's own weights for that entity: unscaled, as a + batch's engine reads them, so :func:`pool_shares` gives each batch + ``c_b = S_b / S``. An input the table does not carry counts as zero, the + engine's default. Check (a) holds every batch's engine to these totals + (:func:`check_key_sums`), so a key the engine reads otherwise is refused + there. + """ + + table = bundle.table(household) + weights = np.asarray(bundle.weights_for(household).values, dtype=np.float64) + products: dict[str, np.ndarray] = {} + for key, inputs in declaration.key_adds.items(): + values = np.zeros(len(table), dtype=np.float64) + for column in inputs: + if column in table.columns: + values = values + table[column].to_numpy(dtype=np.float64) + products[key] = values * weights + return { + batch.key: { + formula: float(np.sum(products[key][batch.households])) + for formula, key in sorted(declaration.corrected.items()) + } + for batch in batches + } + + +def _frame_stores(bundle: Any, name: str, entity: str) -> bool: + """Whether the frame supplies ``name``: a column of its entity's table, + which the engine loads as an input at every year it projects to.""" + + return entity in bundle.entities and name in bundle.table(entity).columns + + +def _judge_requests( + declaration: PopulationDeclaration, + policy: PassPolicy, + *, + requested: Sequence[str], + period: int, + periods: Sequence[int], + stored: Callable[[str], bool], +) -> PopulationVerdict: + """What :func:`judge` would find at every batch's end that the call + itself shows, judged before any engine is built: a period the batches + compute at, or the one asked for, in the window; a refused formula asked + for; a stored formula the frame does not supply, where the pass has no + inert rule for it; and, where it has one, a request for the formula, its + levels or its carriers.""" + + window = declaration.window + violations: list[Violation] = [] + + def flag(kind: str, name: str | None = None, at: Any = None) -> None: + violation = Violation(kind, name, None if at is None else _period_key(at)) + if violation not in violations: + violations.append(violation) + + for at in sorted(set(periods)): + if window.contains(at): + flag("window_period", None, at) + for name in requested: + if window.contains(period): + flag("window_request", name, period) + if name in declaration.refused: + flag("refused_request", name, period) + for name in sorted(declaration.stored): + if stored(name): + continue + rule = declaration.inert.get(name) if policy.inert_allowed else None + if rule is None: + for at in sorted(set(periods)): + flag("stored_not_input", name, at) + continue + unrequestable = rule.unrequestable(name) + for requested_name in requested: + if requested_name in unrequestable: + flag("inert_request", requested_name, period) + verdict = PopulationVerdict( + pass_name=policy.name, + violations=tuple(violations[:_VIOLATION_LIMIT]), + truncated=len(violations) > _VIOLATION_LIMIT, + classes={ + name: () for name in ("corrected", "stored", "zero_at_policy", "inert") + }, + window_first_year=window.first_year, + declaration_sha256=declaration.sha256, + engine=declaration.distribution, + engine_version=_engine_version(declaration.distribution), + ) + if not verdict.clean and policy.on_violation == "raise": + raise PopulationGuardError(verdict) + return verdict + + +def _engine_rows(frame: Any, entity: str) -> np.ndarray: + """The ids of ``entity``'s rows in the order a policyengine-core engine + built from ``frame`` returns them: people as the frame holds them, and + groups by ascending id, as a frame keeps its group tables.""" + + return frame.table(entity)[frame.schema.entity_id_column(entity)].to_numpy() + + +def _run_batch( + frame: Any, + shares: Mapping[str, PoolShare], + names: Sequence[str], + *, + period: int, + dataset_year: int, + declaration: PopulationDeclaration, + policy: PassPolicy, + build: Callable[[Any], Any], +) -> tuple[dict[str, np.ndarray], dict[str, float | None]]: + """One batch: its engine, an :class:`EngineBlock` on it, the requested + values and the block's verdict, and the engine released on the way out, + whether the batch passed or raised.""" + + simulation = build(frame) + block = None + try: + block = EngineBlock( + simulation, + shares, + declaration=declaration, + policy=policy, + periods=(dataset_year, period), + dataset_year=dataset_year, + ) + values: dict[str, np.ndarray] = {} + for name in names: + block.note_request(name, period) + values[name] = np.asarray(simulation.calculate(name, period)) + block.close() + return values, dict(block.growth_factors.get(str(period), {})) + finally: + # One engine alive at a time: nothing here holds this one once the + # batch is done, and the collector frees its cycles before the next + # is built. + del simulation, block + gc.collect() + + +def _refuse_unequal_growth( + growth_by_batch: Mapping[Any, Mapping[str, float | None]], +) -> None: + spread = factor_spread(growth_by_batch) + apart = sorted( + name for name, value in spread.items() if not value <= BATCH_FACTOR_AGREEMENT + ) + if apart: + raise PoolShareError( + f"The batches' engines grow {apart} by different factors, so the " + "shares formed from the dataset year's totals do not hold at the " + "period asked for." + ) + + +def _alike(parts: list[tuple[np.ndarray, np.ndarray]]) -> list[tuple[Any, Any]]: + """The batches' values in one type: text whose batches differ only in + width takes the widest, as one engine's array would; anything else is + left for :func:`scatter_by_entity_ids` to refuse.""" + + dtypes = {values.dtype for _ids, values in parts} + if len(dtypes) > 1 and all(dtype.kind == "U" for dtype in dtypes): + common = np.result_type(*dtypes) + return [(ids, values.astype(common)) for ids, values in parts] + return parts + + +def materialize_in_batches( + bundle: Any, + variables: Sequence[str], + period: int, + *, + batches: Sequence[HouseholdBatch], + declaration: PopulationDeclaration, + policy: PassPolicy, + dataset_year: int, + build: Callable[[Any], Any], + entity_of: Callable[[str], str], + household: str = "household", +) -> dict[str, np.ndarray]: + """``variables`` at ``period`` over ``bundle``, one engine per batch. + + ``batches`` partition the bundle's households (:func:`household_batches`), + ``build`` makes an engine from a batch's frame, ``entity_of`` names a + variable's entity, and the bundle's values are ``dataset_year`` inputs + the engine projects to ``period``. Before any engine is built, the call's + requests are judged (a period in the window, a refused formula, a stored + formula the frame does not supply) and every batch's + :class:`PoolShare` is formed from the frame, ``c_b = S_b / S`` at its + own weights (:func:`frame_key_sums`). Then, batch by batch in the + batches' order: the batch's frame (``bundle.select``), its engine, an + :class:`EngineBlock` on it at ``dataset_year`` and ``period`` (checks + (a) and (b), the pool's shares injected, the recorder armed), the + requested values, the block's verdict, and the engine released before + the next is built. Every batch's engine must grow each key and the + weights by one factor (:data:`BATCH_FACTOR_AGREEMENT`), or the + dataset-year shares would not hold at ``period``. + + The values come back as one engine over the bundle returns them: people + in the bundle's order and groups by ascending id + (:func:`scatter_by_entity_ids`). They are plain arrays, with nothing to + mark them inexact, so the pass must raise: a refusal or a violation in + any batch stops the call. + """ + + if policy.on_violation != "raise": + raise ValueError( + "A batched call returns its values as one engine's, with nothing to " + "mark them inexact, so it runs under a pass that raises." + ) + if len(batches) < 2: + raise ValueError( + "A batched call needs two batches or more; one batch is the frame." + ) + year, dataset = int(period), int(dataset_year) + if dataset > year: + raise ValueError( + f"The dataset's year {dataset} is after {year}; the engine projects " + "a dataset forward only." + ) + names = tuple(dict.fromkeys(str(name) for name in variables)) + entities = {name: str(entity_of(name)) for name in names} + _judge_requests( + declaration, + policy, + requested=names, + period=year, + periods=(dataset, year), + stored=lambda name: _frame_stores(bundle, name, str(entity_of(name))), + ) + shares = pool_shares( + frame_key_sums(bundle, declaration, batches, household=household), + key_variables=dict(declaration.corrected), + ) + parts: dict[str, list[tuple[np.ndarray, np.ndarray]]] = {name: [] for name in names} + growth: dict[Any, dict[str, float | None]] = {} + for batch in batches: + frame = bundle.select(batch.persons) + values, growth[batch.key] = _run_batch( + frame, + shares[batch.key], + names, + period=year, + dataset_year=dataset, + declaration=declaration, + policy=policy, + build=build, + ) + _refuse_unequal_growth(growth) + for name in names: + ids = _engine_rows(frame, entities[name]) + if values[name].shape != ids.shape: + raise ValueError( + f"A batch's engine returns {name!r} for other rows than its " + f"{entities[name]} table's." + ) + parts[name].append((ids, values[name])) + del frame, values + return { + name: scatter_by_entity_ids(bundle, entities[name], _alike(parts[name])) + for name in names + } diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py index a5c3942bc1..04d36be3d0 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py @@ -3,6 +3,11 @@ The PolicyEngine-UK import is deferred until a method needs engine metadata or simulation. Importing this module therefore keeps ``microcosm-frame`` usable without the UK extra. + +A frame whose households carry more than one clone of the dense spine's +geography (``UK_SPINE_ENGINE_BATCH_COLUMN``) is computed clone by clone, one +simulation per clone, through the shared engine-block machinery +(microcosm#931); every other frame takes one simulation, as it always has. """ from __future__ import annotations @@ -19,9 +24,19 @@ import numpy as np +from microcosm.frame.adapters.engine_population import ( + SPINE_BATCHED_PASS, + HouseholdBatch, + household_batches, + materialize_in_batches, +) from microcosm.frame.adapters.policyengine_uk_concepts import ( POLICYENGINE_UK_CONCEPT_MAPPING, ) +from microcosm.frame.adapters.policyengine_uk_population import ( + UK_ENGINE_POPULATION, + UK_SPINE_ENGINE_BATCH_COLUMN, +) from microcosm.frame.bundle import Frame from microcosm.frame.concept_mapping import ConceptMapping from microcosm.frame.materialize import engine_tables @@ -373,12 +388,30 @@ def materialize( class PolicyEngineUKEngine: - """RulesEngine adapter backed by ``policyengine_uk``.""" + """RulesEngine adapter backed by ``policyengine_uk``. + + Whether a call runs in batches is decided from the frame, never set + here: one whose household table carries ``UK_SPINE_ENGINE_BATCH_COLUMN`` + with more than one clone is computed clone by clone + (:meth:`simulation_batches`), since one simulation over a dense spine's + pool does not fit in memory (microcosm#931). ``clones``, the spine's + clone count when known, only refuses: a frame carrying the column with + any other number of clones is part of a pool, or not this spine's, and + an engine over it would read its policyengine-uk shares over the wrong + population. + """ country = "uk" - def __init__(self) -> None: + def __init__(self, *, clones: int | None = None) -> None: + if clones is not None and ( + isinstance(clones, bool) or not isinstance(clones, int) or clones < 1 + ): + raise ValueError("clones is a positive whole number of clones, or None.") self._system: Any = None + self._clones = clones + # The batch count of each call that ran in batches (batch_record). + self._batch_counts: list[int] = [] def variable_metadata(self, name: str) -> VariableMetadata: """Return entity, dtype kind, and period semantics for a UK variable.""" @@ -427,8 +460,16 @@ def materialize( then stamped with that year and policyengine-uk projects it forward to ``period`` as it does a released file (``extend_single_year_dataset``). - The simulation reads only ``period``, so it holds no later year + Every simulation it builds, one over the bundle or one per clone, + reads only ``period``, so it holds no later year (:meth:`_read_year_simulation`). + + A bundle carrying more than one clone (:meth:`simulation_batches`) is + computed clone by clone (:func:`materialize_in_batches`): each clone's + simulation reads the pool's shares (``c_b = S_b / S`` at the frame's + own weights), the UK declaration's guard judges what it ran under the + spine's pass, which raises on anything only the pool defines, and the + values come back as one simulation over the bundle returns them. """ microsimulation_class = self._import_policyengine_uk().Microsimulation @@ -437,6 +478,16 @@ def materialize( f"dataset_period {dataset_period} is after period {period}; " "policyengine-uk projects a dataset forward only." ) + batches = self._clone_batches(bundle) + if batches is not None: + return self._materialize_in_batches( + bundle, + variables, + period, + dataset_period=dataset_period, + batches=batches, + microsimulation_class=microsimulation_class, + ) dataset = self._build_dataset( bundle, period if dataset_period is None else dataset_period ) @@ -455,6 +506,110 @@ def materialize( results[name] = values return results + def simulation_batches(self, bundle: Frame) -> int: + """How many simulations :meth:`materialize` runs for ``bundle``. + + One, or one per clone where the bundle's households carry more than + one; the same decision, and the same refusals, as materialize's. No + engine is built. + """ + + batches = self._clone_batches(bundle) + return 1 if batches is None else len(batches) + + def parameters(self, period: int | str) -> Any: + """The engine's own parameters at ``period``. + + Read from the tax-benefit system every simulation this adapter builds + starts from, so a stage that restates an engine formula over the pool + takes the engine's parameters rather than a copy (microcosm#931). + """ + + return self._tax_benefit_system().parameters(period) + + def batch_record(self) -> dict[str, object] | None: + """How this adapter's calls ran in batches, or None when none did. + + The column the batches follow, how many calls ran in batches, the + batch counts they ran in, and the pass and declaration every batch + was judged under (each was clean: a violation raises). Names and + counts of calls and clones only, never a record's value or a count + of records. + """ + + if not self._batch_counts: + return None + return { + "by": UK_SPINE_ENGINE_BATCH_COLUMN, + "batched_calls": len(self._batch_counts), + "batches": sorted(set(self._batch_counts)), + "pass": SPINE_BATCHED_PASS.name, + "declaration_sha256": UK_ENGINE_POPULATION.sha256, + } + + def _clone_batches(self, bundle: Frame) -> tuple[HouseholdBatch, ...] | None: + """The clone batches a call over ``bundle`` runs in, or None for one + simulation: a bundle without the clone column, or with one clone, + runs as it always has.""" + + entities = getattr(bundle, "entities", None) + if entities is not None and "household" not in entities: + return None + household = bundle.table("household") + column = UK_SPINE_ENGINE_BATCH_COLUMN + if column not in household.columns: + return None + values = household[column] + if values.isna().any(): + raise ValueError( + f"{column!r} is missing for some households, so the clone each " + "belongs to is not known." + ) + clones = int(values.nunique()) + if self._clones is not None and clones != self._clones: + raise ValueError( + f"The frame carries {clones} clone(s) of the spine's {self._clones}. " + "A call over part of the pool would read policyengine-uk's " + "shares over that part, so no engine call runs per clone or " + "over a sub-pool." + ) + if clones < 2: + return None + return household_batches(bundle, column) + + def _materialize_in_batches( + self, + bundle: Frame, + variables: Sequence[str], + period: int | str, + *, + dataset_period: int | str | None, + batches: tuple[HouseholdBatch, ...], + microsimulation_class: Any, + ) -> dict[str, np.ndarray]: + dataset_year = int(period if dataset_period is None else dataset_period) + + def build(frame: Frame) -> Any: + simulation = self._read_year_simulation( + microsimulation_class, self._build_dataset(frame, dataset_year), period + ) + validate_uc_claimant_input(simulation, frame.table("person"), period) + return simulation + + results = materialize_in_batches( + bundle, + variables, + int(period), + batches=batches, + declaration=UK_ENGINE_POPULATION, + policy=SPINE_BATCHED_PASS, + dataset_year=dataset_year, + build=build, + entity_of=lambda name: self.variable_metadata(name).entity, + ) + self._batch_counts.append(len(batches)) + return results + def export_contract(self) -> ExportContract: """UK export contracts are owned by microcosm-build for this release.""" diff --git a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_batches.py b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_batches.py new file mode 100644 index 0000000000..e3d38246f9 --- /dev/null +++ b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_batches.py @@ -0,0 +1,417 @@ +"""policyengine-uk calls over more than one clone, clone by clone (microcosm#931). + +A frame whose households carry the clone index with more than one clone runs +one simulation per clone through the adapter: each clone's simulation reads +the pool's share denominator (``c_b = S_b / S`` at the frame's own weights), +the UK declaration's guard judges it under the spine's pass, and the values +come back in the frame's order. Held here against one simulation over the +same frame without the column (the adapter's single-simulation path), at the +dataset's year and projected a year on: the share formulas within the valid +float32 bound, household taxes within its per-row form, household-local +values bitwise, people in the frame's order and groups by id. What only the +pool defines is refused before any simulation is built, and a frame with one +clone takes the single simulation it always has. Every household here is made +up. +""" + +from __future__ import annotations + +import warnings +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.frame.adapters import policyengine_uk as adapter_module +from microcosm.frame.adapters.engine_population import ( + SPINE_BATCHED_PASS, + PopulationGuardError, + frame_key_sums, + household_batches, + pool_share_atol, + pool_share_rtol, +) +from microcosm.frame.adapters.policyengine_uk import UK_SCHEMA, PolicyEngineUKEngine +from microcosm.frame.adapters.policyengine_uk_population import ( + UK_ENGINE_POPULATION, + UK_SPINE_ENGINE_BATCH_COLUMN, +) + +D, P = 2024, 2025 +CLONES, HOUSEHOLDS = 3, 12 +COICOP = UK_ENGINE_POPULATION.key_adds["consumption"] +REGIONS = ("LONDON", "NORTH_WEST", "SCOTLAND", "WALES", "SOUTH_EAST") +MONTHS = "months_since_last_birthday" +ATTENDANCE = "attends_private_school" + +#: Each share result and the allocation key it divides by: business rates +#: and corporate SDLT are shareholding times a receipt. +SHARES = { + "shareholding": "corporate_sector_wealth", + "corporate_land_value": "corporate_sector_wealth", + "consumption_shareholding": "consumption", + "business_rates": "corporate_sector_wealth", + "corporate_sdlt": "corporate_sector_wealth", +} +SHARE_DEPENDENT_TAXES = ("business_rates", "corporate_sdlt") +#: What a household's own records decide, at each entity, ids included. +LOCAL = ( + "person_id", + "income_tax", + "national_insurance", + "employment_income", + "age_under_18", + "benunit_id", + "benunit_tax", + "num_adults", + "universal_credit", + "LHA_category", + "household_id", + "household_market_income", + "household_benefits", + "household_weight", +) +REQUESTED = (*SHARES, "household_tax", *LOCAL) + + +def _tables(store): + """Three clones of the same households, each drawing its own keys, pay + and weights. Household ids interleave the clones, benefit unit ids sort + the other way, and the people are listed out of order.""" + + persons, households = [], [] + for clone in range(CLONES): + for h in range(HOUSEHOLDS): + household_id = 10 * h + clone + 1 + benunit_id = 5_000 - 2 * household_id + draw = np.random.default_rng(1009 * clone + 17 * h) + shared = np.random.default_rng(5_000 + h) + row = { + "household_id": household_id, + UK_SPINE_ENGINE_BATCH_COLUMN: clone, + "weight": 40_000.0 * (1 + h % 5) / CLONES * draw.uniform(0.7, 1.3), + "region": REGIONS[h % len(REGIONS)], + "council_tax": 1_400.0 + 90 * (h % 5), + "rent": 9_000.0 if h % 2 else 0.0, + "tenure_type": "RENT_PRIVATELY" if h % 2 else "OWNED_OUTRIGHT", + "corporate_wealth": float(draw.lognormal(10, 1.6)) if h % 2 else 0.0, + "private_pension_wealth": ( + float(draw.lognormal(11, 1.1)) if h % 3 else 0.0 + ), + } + for column in COICOP: + row[column] = float(draw.uniform(300, 6_000)) + households.append(row) + for p in range(1 + h % 4): + age = (41, 38, 12, 9)[p] + pay = np.random.default_rng(31 * h + p + 997 * clone) + person = { + "person_id": 10 * household_id + p, + "person_benunit_id": benunit_id, + "person_household_id": household_id, + "age": age, + "gender": "MALE" if (h + p) % 2 else "FEMALE", + "employment_income": ( + float(pay.uniform(0, 60_000)) if age >= 18 and h % 3 else 0.0 + ), + "attends_private_school_random_draw": float(shared.uniform()), + } + if MONTHS in store: + person[MONTHS] = float(shared.integers(0, 12)) + if ATTENDANCE in store: + person[ATTENDANCE] = bool(age < 18 and shared.uniform() < 0.4) + persons.append(person) + person = pd.DataFrame(persons).sample(frac=1.0, random_state=7) + household = pd.DataFrame(households).sort_values("household_id") + benunit = pd.DataFrame( + {"benunit_id": np.sort(person["person_benunit_id"].unique())} + ) + return ( + person.reset_index(drop=True), + benunit, + household.reset_index(drop=True), + ) + + +def _frame(*, clones=True, single=False, store=(MONTHS,)): + person, benunit, household = _tables(store) + weights = household.pop("weight").to_numpy() + if single: + household[UK_SPINE_ENGINE_BATCH_COLUMN] = 0 + if not clones: + household = household.drop(columns=[UK_SPINE_ENGINE_BATCH_COLUMN]) + return Frame( + tables={"person": person, "benunit": benunit, "household": household}, + schema=UK_SCHEMA, + weights={"household": Weights(values=weights, kind=WeightKind.DESIGN)}, + ) + + +def _quiet(call, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return call(*args, **kwargs) + + +def _household_tax_terms(frame, period, dataset_period): + """The pool's household tax terms, mapped to households, for the per-row + bound on household_tax.""" + + from policyengine_uk import Microsimulation + from policyengine_uk.variables.gov.hmrc.household_tax import ( + HOUSEHOLD_TAX_VARIABLES, + ) + + simulation = _quiet( + Microsimulation, + dataset=PolicyEngineUKEngine()._build_dataset(frame, dataset_period), + ) + return { + name: np.asarray( + _quiet(simulation.calculate, name, period, map_to="household"), + dtype=np.float64, + ) + for name in HOUSEHOLD_TAX_VARIABLES + } + + +@pytest.fixture(scope="module") +def system(): + from policyengine_uk import CountryTaxBenefitSystem + + return CountryTaxBenefitSystem() + + +@pytest.fixture( + scope="module", params=[(D, None), (P, D)], ids=["dataset_year", "projected"] +) +def runs(request): + period, dataset_period = request.param + frame, pool = _frame(), _frame(clones=False) + engine = PolicyEngineUKEngine() + batched = _quiet( + engine.materialize, frame, REQUESTED, period, dataset_period=dataset_period + ) + single = _quiet( + PolicyEngineUKEngine().materialize, + pool, + REQUESTED, + period, + dataset_period=dataset_period, + ) + dataset_year = period if dataset_period is None else dataset_period + return SimpleNamespace( + frame=frame, + engine=engine, + batched=batched, + single=single, + period=period, + years=period - dataset_year, + terms=_household_tax_terms(pool, period, dataset_year), + ) + + +def _share_rtol(name, years): + return pool_share_rtol( + HOUSEHOLDS, + CLONES * HOUSEHOLDS, + key_inputs=len(UK_ENGINE_POPULATION.key_adds[SHARES[name]]), + projection_years=years, + ) + + +def test_the_clones_hold_different_shares_of_the_keys(): + # Uncorrected, each clone's simulation would divide by its own totals: + # the shares the data gives each clone are well off a third. + frame = _frame() + batches = household_batches(frame, UK_SPINE_ENGINE_BATCH_COLUMN) + sums = frame_key_sums(frame, UK_ENGINE_POPULATION, batches) + for formula in UK_ENGINE_POPULATION.corrected: + pool = sum(sums[batch.key][formula] for batch in batches) + shares = [sums[batch.key][formula] / pool for batch in batches] + assert max(abs(share - 1 / CLONES) for share in shares) > 0.01, formula + + +def test_share_results_are_the_pools_to_float32_rounding(runs): + for name in SHARES: + batched, single = runs.batched[name], runs.single[name] + assert batched.dtype == single.dtype, name + assert np.any(single != 0), name + np.testing.assert_allclose( + batched, + single, + rtol=_share_rtol(name, runs.years), + atol=0, + err_msg=f"{name} at {runs.period}", + ) + + +def test_household_taxes_are_the_pools_within_the_per_row_bound(runs): + terms = runs.terms + atol = pool_share_atol( + sum(np.abs(terms[name]) for name in SHARE_DEPENDENT_TAXES), + sum(np.abs(values) for values in terms.values()), + share_rtol=max(_share_rtol(name, runs.years) for name in SHARE_DEPENDENT_TAXES), + additions=len(terms), + ) + gap = np.abs( + runs.batched["household_tax"].astype(np.float64) + - runs.single["household_tax"].astype(np.float64) + ) + assert np.all(gap <= atol), f"household_tax at {runs.period}" + assert np.any(terms["business_rates"] != 0) + + +def test_household_local_values_are_the_pools_bitwise(runs): + for name in LOCAL: + batched, single = runs.batched[name], runs.single[name] + assert type(batched) is type(single), name + assert batched.dtype == single.dtype, name + np.testing.assert_array_equal(batched, single, err_msg=name) + assert np.any(runs.single["income_tax"] != 0) + + +def test_people_come_back_in_the_frames_order_and_groups_by_id(runs): + frame = runs.frame + for entity, name in ( + ("person", "person_id"), + ("benunit", "benunit_id"), + ("household", "household_id"), + ): + np.testing.assert_array_equal( + runs.batched[name], frame.table(entity)[name].to_numpy(), err_msg=entity + ) + # The frame lists people out of id order and benefit units against the + # households' order, so neither is the batches' concatenation. + people = frame.table("person")["person_id"].to_numpy() + assert not np.array_equal(people, np.sort(people)) + + +def test_the_call_ran_in_three_judged_batches(runs): + assert runs.engine.simulation_batches(runs.frame) == CLONES + assert runs.engine.batch_record() == { + "by": UK_SPINE_ENGINE_BATCH_COLUMN, + "batched_calls": 1, + "batches": [CLONES], + "pass": SPINE_BATCHED_PASS.name, + "declaration_sha256": UK_ENGINE_POPULATION.sha256, + } + + +def test_a_frame_with_one_clone_takes_one_simulation_bitwise(monkeypatch): + def refuse(*args, **kwargs): + raise AssertionError("a frame with one clone never runs in batches") + + monkeypatch.setattr(adapter_module, "materialize_in_batches", refuse) + engine = PolicyEngineUKEngine() + one = _quiet( + engine.materialize, _frame(single=True), REQUESTED, P, dataset_period=D + ) + plain = _quiet( + PolicyEngineUKEngine().materialize, + _frame(clones=False), + REQUESTED, + P, + dataset_period=D, + ) + for name in REQUESTED: + assert type(one[name]) is type(plain[name]), name + assert one[name].dtype == plain[name].dtype, name + np.testing.assert_array_equal(one[name], plain[name], err_msg=name) + assert engine.simulation_batches(_frame(single=True)) == 1 + assert engine.batch_record() is None + + +def test_stored_attendance_may_be_asked_for_in_batches(): + store = (MONTHS, ATTENDANCE) + frame = _frame(store=store) + batched = _quiet( + PolicyEngineUKEngine().materialize, + frame, + [ATTENDANCE, "household_tax"], + P, + dataset_period=D, + ) + stored = frame.table("person")[ATTENDANCE].to_numpy() + assert stored.any() + np.testing.assert_array_equal(batched[ATTENDANCE], stored) + + +def _refusing_engine(monkeypatch, system, **kwargs): + """An adapter that refuses to build any simulation.""" + + def refuse(**_kwargs): + raise AssertionError("no simulation is built") + + engine = PolicyEngineUKEngine(**kwargs) + engine._system = system + monkeypatch.setattr( + engine, + "_import_policyengine_uk", + lambda: SimpleNamespace(Microsimulation=refuse), + ) + return engine + + +@pytest.mark.parametrize( + ("names", "period", "store", "expected"), + [ + ( + ["income_tax", ATTENDANCE], + P, + (MONTHS,), + {("inert_request", ATTENDANCE, str(P))}, + ), + ( + ["income_tax", "household_income_decile"], + P, + (MONTHS,), + {("refused_request", "household_income_decile", str(P))}, + ), + ( + ["income_tax"], + 2026, + (MONTHS,), + { + ("window_period", None, "2026"), + ("window_request", "income_tax", "2026"), + }, + ), + ( + ["income_tax"], + P, + (), + { + ("stored_not_input", MONTHS, str(D)), + ("stored_not_input", MONTHS, str(P)), + }, + ), + ], + ids=["attendance_unstored", "decile", "rebalancing_year", "months_unstored"], +) +def test_what_only_the_pool_defines_is_refused_before_any_simulation( + monkeypatch, system, names, period, store, expected +): + engine = _refusing_engine(monkeypatch, system) + with pytest.raises(PopulationGuardError) as raised: + engine.materialize(_frame(store=store), names, period, dataset_period=D) + found = {(v.kind, v.name, v.period) for v in raised.value.verdict.violations} + assert found == expected + assert engine.batch_record() is None + + +def test_a_frame_with_fewer_clones_than_the_spine_is_refused(monkeypatch, system): + engine = _refusing_engine(monkeypatch, system, clones=CLONES + 1) + with pytest.raises(ValueError, match="no engine call runs per clone"): + engine.materialize(_frame(), ["income_tax"], P, dataset_period=D) + with pytest.raises(ValueError, match="no engine call runs per clone"): + engine.simulation_batches(_frame(single=True)) + + +def test_the_adapters_parameters_are_the_engines(system): + engine = PolicyEngineUKEngine() + node = engine.parameters(P).gov.simulation.private_school_vat + expected = system.parameters(P).gov.simulation.private_school_vat + assert node.private_school_factor == expected.private_school_factor diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py b/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py index 4983a29151..f45acf2487 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py @@ -10,9 +10,11 @@ from __future__ import annotations +import gc import math import sys import types +import weakref from types import SimpleNamespace import numpy as np @@ -21,6 +23,7 @@ from microcosm.frame import EntitySchema, Frame, WeightKind, Weights from microcosm.frame.adapters.engine_population import ( + BATCH_FACTOR_AGREEMENT, FLOAT32_UNIT_ROUNDOFF, MEASURE_PASS, SPINE_BATCHED_PASS, @@ -42,9 +45,11 @@ derived_watched_periods, factor_spread, float32_sum_roundings, + frame_key_sums, household_batches, inject_pool_shares, judge, + materialize_in_batches, pool_share_atol, pool_share_rtol, pool_shares, @@ -58,9 +63,11 @@ FAKE_ENGINE, GROWTH, KEY_A, + KEY_B, WEIGHT, D, FakeSimulation, + FakeVariable, N, P, install_fake_engine, @@ -1717,3 +1724,271 @@ def test_scatter_refuses_missing_repeated_or_mistyped_rows(): "household", [(ids[:3], values[:3]), (ids[3:], values[3:].astype(np.float64))], ) + + +# --------------------------------------------------------------------------- +# One call in batches +# --------------------------------------------------------------------------- + +#: Two clones of the fakes' six households, interleaved by id, so a batch's +#: rows are never a run of the frame's. +_CLONE_OF = np.array([0, 1, 0, 1, 1, 0]) +_IDS = np.arange(10, 10 + N, dtype=np.int64) + + +def _batched_frame(*, store=("months", "attendance"), keys=("key_a", "key_b")): + """The fakes' households under two interleaved clones, one person each, + the people listed out of id order. The fake engine's variables are all + household ones, so the household table holds what it loads.""" + + household = pd.DataFrame({"household_id": _IDS, "clone": _CLONE_OF}) + if "key_a" in keys: + household["key_a"] = KEY_A + if "key_b" in keys: + household["key_b"] = KEY_B + if "months" in store: + household["months"] = np.full(N, 3.0) + if "attendance" in store: + household["attendance"] = np.zeros(N, dtype=bool) + order = np.array([4, 0, 5, 2, 1, 3]) + person = pd.DataFrame( + { + "person_id": _IDS[order] * 10, + "person_benunit_id": _IDS[order], + "person_household_id": _IDS[order], + } + ) + return Frame( + { + "person": person, + "benunit": pd.DataFrame({"benunit_id": _IDS}), + "household": household, + }, + EntitySchema(group_entities=("benunit", "household")), + {"household": Weights(WEIGHT.copy(), kind=WeightKind.DESIGN)}, + ) + + +def _engine_of(frame, *, system=None, growth=GROWTH, key_scale=1.0): + """A fake engine over a frame: its keys at the dataset's year and grown + by ``growth`` a year on, the frame's own weights, and what it stores.""" + + household = frame.table("household") + inputs = {} + for period, factor in ((D, 1.0), (P, growth)): + for name, scale in (("key_a", key_scale), ("key_b", 1.0)): + if name in household: + inputs[(name, period)] = household[name].to_numpy() * factor * scale + inputs[("weight", period)] = frame.weights_for("household").values + for name in ("months", "attendance"): + if name in household: + inputs[(name, period)] = household[name].to_numpy() + return FakeSimulation( + _fake_system() if system is None else system, inputs, rows=len(household) + ) + + +def _in_batches(frame, names, *, period=P, build=None, policy=SPINE_BATCHED_PASS): + return materialize_in_batches( + frame, + names, + period, + batches=household_batches(frame, "clone"), + declaration=_declaration(), + policy=policy, + dataset_year=D, + build=_engine_of if build is None else build, + entity_of=lambda name: "household", + ) + + +@pytest.mark.parametrize("period", [D, P]) +def test_a_call_in_batches_gives_one_engines_values_in_the_frames_order( + fake_engine, period +): + frame = _batched_frame() + built = [] + + def build(batch_frame): + # One engine alive at a time: the last batch's is gone. + assert all(reference() is None for reference in built) + ids = batch_frame.table("household")["household_id"].to_numpy() + assert len(set(_CLONE_OF[np.isin(_IDS, ids)])) == 1 + simulation = _engine_of(batch_frame) + # A reference cycle, as policyengine-uk's system keeps to its + # simulation: only a collection frees it. + simulation.tax_benefit_system = SimpleNamespace( + variables=simulation.tax_benefit_system.variables, + parameters=simulation.tax_benefit_system.parameters, + simulation=simulation, + ) + built.append(weakref.ref(simulation)) + return simulation + + automatic = gc.isenabled() + gc.disable() + try: + values = _in_batches( + frame, ["share", "local", "rates"], period=period, build=build + ) + finally: + if automatic: + gc.enable() + assert len(built) == 2 + pool = _engine_of(frame) + # The corrected share is the pool's to float32 rounding; a household-local + # value is the pool engine's, bitwise; both in the frame's household order. + np.testing.assert_allclose( + values["share"], pool.calculate("share", period), rtol=1e-6 + ) + np.testing.assert_allclose( + values["rates"], pool.calculate("rates", period), rtol=1e-6 + ) + np.testing.assert_array_equal(values["local"], pool.calculate("local", period)) + assert values["share"].dtype == pool.calculate("share", period).dtype + # Uncorrected, the clones' own shares are not the pool's. + own = np.empty(N) + for batch in household_batches(frame, "clone"): + own[batch.households] = _engine_of(frame.select(batch.persons)).calculate( + "share", period + ) + assert not np.allclose(own, pool.calculate("share", period), rtol=1e-3) + + +def test_frame_key_sums_read_the_declared_inputs_at_the_frames_weights(): + frame = _batched_frame() + batches = household_batches(frame, "clone") + sums = frame_key_sums(frame, _declaration(), batches) + for batch in batches: + rows = batch.households + assert sums[batch.key] == { + "share": pytest.approx(float(np.sum((KEY_A + KEY_B)[rows] * WEIGHT[rows]))) + } + # An input the frame does not carry is the engine's default, zero. + sums = frame_key_sums(_batched_frame(keys=("key_a",)), _declaration(), batches) + for batch in batches: + rows = batch.households + expected = float(np.sum(KEY_A[rows] * WEIGHT[rows])) + assert sums[batch.key]["share"] == pytest.approx(expected) + + +def _refused(frame, names, *, period=P, store_check=None): + calls = [] + + def build(batch_frame): + calls.append(batch_frame) + return _engine_of(batch_frame) + + with pytest.raises(PopulationGuardError) as raised: + _in_batches(frame, names, period=period, build=build) + assert calls == [] + verdict = raised.value.verdict + assert numbers_are_years([str(raised.value)]) + return {(v.kind, v.name, v.period) for v in verdict.violations} + + +def test_a_call_in_batches_is_judged_before_any_engine_is_built(fake_engine): + frame = _batched_frame() + assert ("refused_request", "decile", str(P)) in _refused(frame, ["local", "decile"]) + late = _refused(frame, ["local"], period=2026) + assert ("window_period", None, "2026") in late + assert ("window_request", "local", "2026") in late + # A stored formula the frame does not supply cannot be the pool's. + unstored = _refused(_batched_frame(store=("attendance",)), ["local"]) + assert ("stored_not_input", "months", str(D)) in unstored + assert ("stored_not_input", "months", str(P)) in unstored + # Attendance may stay unstored under the spine's inert rule, but neither + # it, its levels nor its carriers may be asked for then. + for name in ("attendance", "vat", "baseline_vat", "tax_total"): + kinds = _refused(_batched_frame(store=("months",)), ["local", name]) + assert kinds == {("inert_request", name, str(P))} + unstored_attendance = _batched_frame(store=("months",)) + values = _in_batches(unstored_attendance, ["local"]) + np.testing.assert_array_equal( + values["local"], _engine_of(unstored_attendance).calculate("local", P) + ) + + +def test_a_batch_that_runs_what_only_the_pool_defines_raises(fake_engine): + frame = _batched_frame() + + def build(batch_frame): + return _engine_of(batch_frame, system=_fake_system(budget=5.0)) + + with pytest.raises(PopulationGuardError) as raised: + _in_batches(frame, ["budget_change"], build=build) + kinds = {(v.kind, v.name) for v in raised.value.verdict.violations} + assert kinds == {("zero_at_policy", "budget_change")} + + +def test_batches_whose_engines_grow_a_key_apart_are_refused(fake_engine): + frame = _batched_frame() + + def build(batch_frame): + first = _IDS[0] in set(batch_frame.table("household")["household_id"]) + return _engine_of(batch_frame, growth=GROWTH if first else GROWTH * 1.001) + + with pytest.raises(PoolShareError, match=r"grow \['key'\] by different"): + _in_batches(frame, ["share"], build=build) + assert 1.001 - 1 > BATCH_FACTOR_AGREEMENT + + +def test_a_batch_engine_whose_key_total_is_not_the_frames_is_refused(fake_engine): + def build(batch_frame): + return _engine_of(batch_frame, key_scale=1.1) + + with pytest.raises(PoolShareError, match="not the block's frame total"): + _in_batches(_batched_frame(), ["share"], build=build) + + +def test_text_whose_batches_differ_in_width_takes_the_widest(fake_engine): + system = _fake_system() + system.variables["size"] = FakeVariable( + "size", + formula=lambda simulation, period: np.where( + simulation.calculate("key_a", period) > 20, "large", "s" + ), + dtype=str, + ) + values = _in_batches( + _batched_frame(), + ["size"], + build=lambda batch_frame: _engine_of(batch_frame, system=system), + ) + expected = np.where(KEY_A * GROWTH > 20, "large", "s") + np.testing.assert_array_equal(values["size"], expected) + assert values["size"].dtype == expected.dtype + + +def test_a_call_in_batches_needs_a_raising_pass_two_batches_and_no_back_cast( + fake_engine, +): + frame = _batched_frame() + with pytest.raises(ValueError, match="runs under a pass that raises"): + _in_batches(frame, ["local"], policy=MEASURE_PASS) + single = household_batches(frame, "clone")[:1] + with pytest.raises(ValueError, match="two batches or more"): + materialize_in_batches( + frame, + ["local"], + P, + batches=single, + declaration=_declaration(), + policy=SPINE_BATCHED_PASS, + dataset_year=D, + build=_engine_of, + entity_of=lambda name: "household", + ) + with pytest.raises(ValueError, match="projects a dataset forward only"): + materialize_in_batches( + frame, + ["local"], + D, + batches=household_batches(frame, "clone"), + declaration=_declaration(), + policy=SPINE_BATCHED_PASS, + dataset_year=P, + build=_engine_of, + entity_of=lambda name: "household", + ) + gc.collect() diff --git a/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py b/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py index 1e93588009..3bc2574447 100644 --- a/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py +++ b/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py @@ -1,6 +1,13 @@ """Tests split from packages/microcosm-frame/tests/test_policyengine_uk_adapter.py.""" # ruff: noqa: F403, F405 +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.frame.adapters import policyengine_uk as adapter_module +from microcosm.frame.adapters.engine_population import SPINE_BATCHED_PASS +from microcosm.frame.adapters.policyengine_uk_population import ( + UK_ENGINE_POPULATION, + UK_SPINE_ENGINE_BATCH_COLUMN, +) from test_support.microcosm_frame.policyengine_uk_adapter import * @@ -501,3 +508,213 @@ def test_the_whole_loop_block_refuses_a_changed_engine(monkeypatch, constants): with pytest.raises(TypeError, match="FIRST_REBALANCING_YEAR"): with uk_whole_rebalancing_years(): pass + + +# --------------------------------------------------------------------------- +# Calls over more than one clone run clone by clone (microcosm#931) +# --------------------------------------------------------------------------- + + +def _clone_frame(clones=(0, 1, 2), *, column=True, missing=False): + """Two households per clone, one person each, clones interleaved by id. + Everything is made up.""" + + households = len(clones) * 2 + ids = np.arange(1, households + 1, dtype="int64") + household = pd.DataFrame({"household_id": ids}) + if column: + index = np.array([clones[i % len(clones)] for i in range(households)]) + household[UK_SPINE_ENGINE_BATCH_COLUMN] = ( + pd.array(index, dtype="Int64") if missing else index + ) + if missing: + household.loc[0, UK_SPINE_ENGINE_BATCH_COLUMN] = pd.NA + person = pd.DataFrame( + { + "person_id": ids[::-1] * 10, + "person_benunit_id": ids[::-1], + "person_household_id": ids[::-1], + "age": np.full(households, 40.0), + } + ) + return Frame( + tables={ + "person": person, + "benunit": pd.DataFrame({"benunit_id": ids}), + "household": household, + }, + schema=EntitySchema(group_entities=("benunit", "household")), + weights={ + "household": Weights( + values=np.full(households, 100.0), kind=WeightKind.DESIGN + ) + }, + ) + + +def _no_engine(monkeypatch, engine): + def refuse(): + raise AssertionError("no engine is built") + + monkeypatch.setattr(engine, "_import_policyengine_uk", refuse) + + +def test_whether_a_call_batches_is_decided_from_the_frame(monkeypatch): + engine = PolicyEngineUKEngine() + _no_engine(monkeypatch, engine) + assert engine.simulation_batches(_clone_frame(column=False)) == 1 + assert engine.simulation_batches(_clone_frame((0,))) == 1 + assert engine.simulation_batches(_clone_frame((0, 1, 2))) == 3 + assert engine.batch_record() is None + + +@pytest.mark.parametrize( + ("expected", "clones", "batches"), + [ + (3, (0, 1, 2), 3), + (3, (0,), None), + (3, (0, 1), None), + (1, (0,), 1), + (1, (0, 1), None), + ], +) +def test_the_spines_clone_count_refuses_any_other_frame( + monkeypatch, expected, clones, batches +): + # Part of the pool (one clone, or fewer than the spine's) or another + # spine's frame is refused before any engine is built; a frame without + # the column (a stage before the clones exist) runs as it always has. + engine = PolicyEngineUKEngine(clones=expected) + assert engine.simulation_batches(_clone_frame(column=False)) == 1 + frame = _clone_frame(clones) + if batches is not None: + assert engine.simulation_batches(frame) == batches + return + with pytest.raises(ValueError, match="no engine call runs per clone"): + engine.simulation_batches(frame) + monkeypatch.setattr( + engine, + "_import_policyengine_uk", + lambda: SimpleNamespace(Microsimulation=lambda **kwargs: pytest.fail()), + ) + with pytest.raises(ValueError, match="no engine call runs per clone"): + engine.materialize(frame, ["age"], 2025) + + +@pytest.mark.parametrize("clones", [0, -1, True, 2.0, "3"]) +def test_the_spines_clone_count_is_a_positive_whole_number(clones): + with pytest.raises(ValueError, match="positive whole number"): + PolicyEngineUKEngine(clones=clones) + + +def test_a_clone_index_with_missing_households_is_refused(monkeypatch): + engine = PolicyEngineUKEngine() + _no_engine(monkeypatch, engine) + with pytest.raises(ValueError, match="clone each belongs to"): + engine.simulation_batches(_clone_frame((0, 1), missing=True)) + + +def _one_simulation_engine(monkeypatch): + """An adapter whose engine is a stub: one call builds one simulation.""" + + built = [] + + class Simulation: + tax_benefit_system = SimpleNamespace(variables={}) + input_variables = () + + def __init__(self, dataset): + built.append(dataset) + + def calculate(self, name, period=None): + return np.arange(len(built[-1]), dtype=np.float32) + + engine = PolicyEngineUKEngine() + monkeypatch.setattr( + engine, + "_import_policyengine_uk", + lambda: SimpleNamespace(Microsimulation=Simulation), + ) + # The engine's year window needs policyengine-uk itself; the stand-in + # engine is built from the dataset as it is. + monkeypatch.setattr( + engine, + "_read_year_simulation", + lambda microsimulation_class, dataset, period: microsimulation_class( + dataset=dataset + ), + ) + monkeypatch.setattr( + engine, "_build_dataset", lambda bundle, period: list(range(bundle.n("person"))) + ) + monkeypatch.setattr( + engine, + "variable_metadata", + lambda name: SimpleNamespace(entity="person"), + ) + return engine, built + + +@pytest.mark.parametrize("clones", [None, (0,)]) +def test_a_frame_with_one_clone_takes_one_simulation(monkeypatch, clones): + engine, built = _one_simulation_engine(monkeypatch) + + def refuse(*args, **kwargs): + raise AssertionError("one clone never runs in batches") + + monkeypatch.setattr(adapter_module, "materialize_in_batches", refuse) + frame = _clone_frame(column=False) if clones is None else _clone_frame(clones) + values = engine.materialize(frame, ["age"], 2025) + assert len(built) == 1 and len(built[0]) == frame.n("person") + np.testing.assert_array_equal(values["age"], np.arange(frame.n("person"))) + assert engine.batch_record() is None + + +def test_a_frame_with_clones_runs_through_the_shared_batches(monkeypatch): + engine, built = _one_simulation_engine(monkeypatch) + calls = [] + + def batched(bundle, variables, period, **kwargs): + calls.append((bundle, variables, period, kwargs)) + return {"age": np.zeros(bundle.n("person"))} + + monkeypatch.setattr(adapter_module, "materialize_in_batches", batched) + roles = [] + monkeypatch.setattr( + adapter_module, + "validate_uc_claimant_input", + lambda simulation, person, period: roles.append((len(person), period)), + ) + frame = _clone_frame((0, 1, 2)) + engine.materialize(frame, ["age"], 2025, dataset_period=2023) + ((bundle, variables, period, kwargs),) = calls + assert bundle is frame and list(variables) == ["age"] and period == 2025 + assert kwargs["declaration"] is UK_ENGINE_POPULATION + assert kwargs["policy"] is SPINE_BATCHED_PASS + assert kwargs["dataset_year"] == 2023 + assert [batch.key for batch in kwargs["batches"]] == [0, 1, 2] + assert kwargs["entity_of"]("age") == "person" + # Each batch's engine is built from the batch's own frame, at the + # dataset's year, and its roles are checked against the batch's people. + batch = frame.select(kwargs["batches"][1].persons) + kwargs["build"](batch) + assert built == [list(range(batch.n("person")))] + assert roles == [(batch.n("person"), 2025)] + assert engine.batch_record() == { + "by": UK_SPINE_ENGINE_BATCH_COLUMN, + "batched_calls": 1, + "batches": [3], + "pass": SPINE_BATCHED_PASS.name, + "declaration_sha256": UK_ENGINE_POPULATION.sha256, + } + engine.materialize(frame, ["age"], 2025) + assert calls[-1][3]["dataset_year"] == 2025 + assert engine.batch_record()["batched_calls"] == 2 + with pytest.raises(ValueError, match="projects a dataset forward only"): + engine.materialize(frame, ["age"], 2023, dataset_period=2025) + + +def test_the_parameters_are_the_engines_own(): + engine = PolicyEngineUKEngine() + engine._system = SimpleNamespace(parameters=lambda period: ("tree at", period)) + assert engine.parameters(2025) == ("tree at", 2025) From ff8b855e58f60ca5538cc62a479b36505d1cc5d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 13:32:45 +0000 Subject: [PATCH 005/108] Rank private-school attendance over the pool from household-local engine values (#931) policyengine-uk's attends_private_school ranks household net income over the people a simulation holds, so once a spine's engine calls run clone by clone, one clone's simulation would rank that clone alone. Stage 52 now restates the formula's last step itself, over every person in the frame: the members' income tax and National Insurance summed per household as policyengine-core sums them, net income from the household's market income and benefits, the household's weight once per member, microdf's MicroSeries.percentile_rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor from the engine's own parameters (parameters(period), at the two paths PRIVATE_SCHOOL_ATTENDANCE_PARAMETERS names), the one-million floor, and the stored draw for under-18s. Every input is the engine's own value at the calibration year, decided by a household's own records. - Where the engine computes the frame in one simulation (national, and every spine today), the stage asks for the engine's attendance beside those inputs, stores the engine's flags, and raises if the restatement differs from them bit for bit, so each such build re-checks it on its own data. Where the adapter computes the frame clone by clone (simulation_batches above one), it asks for the inputs only and stores the restatement. An engine without simulation_batches runs one. - The evidence gains rank_route (engine_with_parity or pooled_restatement) and engine_parity_checked, a boolean: whether the engine's flags were checked. A failed check raises, so neither records a mismatch. weighted_pupils stays. The parity error names the formula, never a count. - The stage's notes in sources.yaml describe both routes. The notes in source_stages.json are re-derived from it, and the release input coverage manifest is regenerated (its source_manifest_sha256 fields). - Left for G1, which makes the clone-by-clone route live and regenerates the spine fixtures: the stage's source line ("evaluated by the pinned engine"), the notes' sentence that the stage materializes the engine's attendance, and the H2 fixture's sources/fixture.json, which keeps the stage's earlier notes until tools/graph_uk_spine_fixture.py runs. All three hold while every call takes one simulation. The store_rules_engine_input operation describes both routes: each stores attendance as an engine input from the stored draw, at the calibration year and the spine's design weights. No data changes on today's lines: no spine frame carries household_clone_index before G1, so every call takes one simulation and stores the engine's flags, as before. Identities move. Stage 52's kernel hashes private_school_attendance.py, so stage 52 and every node after it re-key. The packaged resources enter the full and national target kernels' country-spec fingerprint, so those re-key again. National's stage-52 evidence gains the two fields. The spine graph declaration (uk_spine.json) is unchanged, since stage contracts do not cover notes. An engine test runs a synthetic pool of three clones of twelve households: some alike in every clone and some not, most with several children, weights spread about one to ten, ties, a household of no weight, and each clone under the floor while the pool is over it. Household ids interleave the clones and the people are listed out of order. Each child's draw sits 3% under or over the rate the formula gives it over the pool, from the formula's own percentiles and the engine's parameters, so a rank, rate, interpolation or factor a few percent off would store other flags. A spy on microdf's percentile_rank shows the restatement hands it the formula's net incomes and weights and gets its percentiles back, bit for bit, in one simulation and clone by clone. The stage run clone by clone stores what one simulation stores, where a clone's own simulation imputes nobody; a population factor of zero, or 6% lower, breaks the parity check; below the floor nobody attends on either route, while design weights just under it impute on both, since the formula reads the engine's weights at the calibration year; and a frame of one clone stores the engine's attendance as the stage asked for it before. Engine-free tests run both routes, the parity refusal and the floor on a fake engine with a stand-in rank. Co-Authored-By: Claude Opus 5.5 --- .../uk/release_input_coverage_manifest.json | 50 +- .../src/microcosm/build/uk/source_stages.json | 2 +- .../src/microcosm/build/uk/spec/sources.yaml | 2 +- .../uk_runtime/private_school_attendance.py | 273 ++++++++- .../uk/test_uk_private_school_pooled_rank.py | 517 ++++++++++++++++++ .../uk/test_uk_private_school_attendance.py | 261 +++++++-- 6 files changed, 1023 insertions(+), 82 deletions(-) create mode 100644 packages/microcosm-build/tests/engine/uk/test_uk_private_school_pooled_rank.py diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 4096c30332..52b0e50179 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 36586bd836..6d2f6b780a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -8398,7 +8398,7 @@ "outputs": [ "attends_private_school" ], - "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine's attendance once, at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine." + "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine's attendance once, at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula's last step over every person in the frame, from the engine's own values at the calibration year, each decided by a household's own records, and its own parameters: household market income plus benefits less the members' income tax and National Insurance, each household's weight once per member, microdf's weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine before its geographic clones exist), the stage stores the engine's flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine's flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine." }, { "artifacts": [ diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 9e36d8ecba..3bc572d399 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -6817,7 +6817,7 @@ stages: gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: - attends_private_school - notes: 'The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine''s attendance once, at the calibration year and the spine''s design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.' + notes: 'The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine''s attendance once, at the calibration year and the spine''s design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula''s last step over every person in the frame, from the engine''s own values at the calibration year, each decided by a household''s own records, and its own parameters: household market income plus benefits less the members'' income tax and National Insurance, each household''s weight once per member, microdf''s weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine before its geographic clones exist), the stage stores the engine''s flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine''s flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.' - stage: isa_subscriptions survey: HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8 source: HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain). diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py index 72483559b8..026fbe563f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py @@ -14,21 +14,37 @@ * the calibration's engine run and the release's read different flags once the weights change. -This stage, the last of the spine, materializes the engine's attendance once, -at the calibration year and the spine's design weights, and stores it, as the -root stage stores ``months_since_last_birthday``. The solve and the release -then read the same flags, and no stage after it copies or re-keys a person. +This stage, the last of the spine, stores attendance once, at the calibration +year and the spine's design weights, as the root stage stores +``months_since_last_birthday``. The solve and the release then read the same +flags, and no stage after it copies or re-keys a person. Stored attendance no longer responds to income changes or to reforms of the attendance parameters; a reform to the fee VAT does not move pupils either way in the pinned engine. + +The rank is over the people a simulation holds, so a simulation over part of +the frame ranks that part alone. Once a dense spine's households carry more +than one geographic clone, its engine calls run clone by clone +(microcosm#931), so the stage restates the formula's last step itself, over +every person in the frame (:func:`restate_private_school_attendance`): from +engine values that each household's own records decide, the engine's own +parameters, and microdf's weighted percentile rank, the rank the formula +calls. Where the engine computes the frame in one simulation (the national +line, and a spine before its clones exist) the stage stores the engine's +flags once the restatement equals them bit for bit, so every such build +re-checks the restatement on its own data; where it computes the frame clone +by clone the stage stores the restatement. The receipt names the route and +whether the engine's flags were checked, never a count. """ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass, field +from typing import Any import numpy as np +import pandas as pd from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime.frs_release import resolve_uk_year_rule @@ -54,6 +70,40 @@ "The terminal family gate requires a valid mass-conserving MassChangeRecord " "carrying exactly this stage-specific reason." ) +#: How the stored flags were reached: the engine's own, once the restatement +#: equalled them bit for bit (one simulation over the frame), or the +#: restatement over the whole frame (the engine computes it clone by clone). +PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE = "engine_with_parity" +PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE = "pooled_restatement" +#: What policyengine-uk's formula reads (2.174.1, +#: variables/contrib/labour/attends_private_school.py, a source the UK engine +#: declaration pins), each decided by a household's own records, so the +#: engine returns it alike over the frame and clone by clone. +PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS = ( + "income_tax", + "national_insurance", + "age_under_18", + PRIVATE_SCHOOL_ATTENDANCE_DRAW, +) +PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS = ( + "household_market_income", + "household_benefits", + "household_weight", +) +#: The engine parameters the formula reads, the only ones the restatement +#: reads: the attendance rate at every fifth percentile, and the population +#: factor. +PRIVATE_SCHOOL_ATTENDANCE_PARAMETERS = ( + "gov.simulation.private_school_vat.private_school_attendance_rate", + "gov.simulation.private_school_vat.private_school_factor", +) +# The formula's own constants: the percentiles its attendance rates are given +# at, the percentiles it looks rates up for, and the total weight of people +# (a household's weight once per member) below which it imputes no attendance +# (pe-uk#1905). The pinned source and the parity check hold these copies to it. +_RATE_PERCENTILES = tuple(range(0, 101, 5)) +_LOOKUP_PERCENTILES = 101 +_IMPUTED_POPULATION_FLOOR = 1e6 @dataclass(frozen=True) @@ -63,6 +113,8 @@ class UKPrivateSchoolAttendanceResult: frame: Frame year: int weighted_pupils: float + rank_route: str + engine_parity_checked: bool def evidence(self) -> dict[str, object]: return { @@ -70,8 +122,15 @@ def evidence(self) -> dict[str, object]: "year_rule": PRIVATE_SCHOOL_ATTENDANCE_YEAR_RULE, "year": self.year, "weights": PRIVATE_SCHOOL_ATTENDANCE_WEIGHTS, - # A national weighted total over every pupil the engine imputes. + # A national weighted total over every pupil the stage stores. "weighted_pupils": self.weighted_pupils, + # The route's name, and whether the engine's own flags were checked + # against the restatement: True over one simulation; False where + # the engine computes the frame clone by clone and no simulation + # holds the pool's flags. A check that fails raises, so the field + # never records a mismatch. + "rank_route": self.rank_route, + "engine_parity_checked": self.engine_parity_checked, } @@ -107,7 +166,15 @@ def checkpoint_metadata(self) -> dict[str, object]: def store_private_school_attendance( frame: Frame, *, engine: object, calibration_year: int | None = None ) -> UKPrivateSchoolAttendanceResult: - """Store the engine's attendance at the calibration year and these weights.""" + """Store policyengine-uk's attendance at the calibration year and these weights. + + ``engine`` computes the frame's values (``materialize``) and gives its own + parameters (``parameters(period)``). Where it computes the frame in one + simulation, the stage stores the engine's flags once the restatement over + the frame equals them bit for bit, and raises otherwise; where it computes + the frame in more (``simulation_batches``: one per clone; an engine + without the method runs one), the stage stores the restatement. + """ validate_uk_national_frame(frame) assert_rules_engine_country(engine, "uk") @@ -128,21 +195,42 @@ def store_private_school_attendance( if calibration_year is not None else resolve_uk_year_rule(PRIVATE_SCHOOL_ATTENDANCE_YEAR_RULE) ) + parameters = _engine_parameters(engine, year) + batched = _simulation_batches(engine, frame) > 1 + inputs = ( + *PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS, + *PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS, + ) + # A call that runs clone by clone refuses the engine's own attendance, which + # would rank each clone alone; one simulation computes it beside the inputs. + requested = inputs if batched else (PRIVATE_SCHOOL_ATTENDANCE_OUTPUT, *inputs) materialized = engine.materialize( - frame, [PRIVATE_SCHOOL_ATTENDANCE_OUTPUT], year, dataset_period=period + frame, list(requested), year, dataset_period=period ) - if PRIVATE_SCHOOL_ATTENDANCE_OUTPUT not in materialized: - raise ValueError( - "Private-school attendance engine output missing: " - f"{[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]}." - ) - values = np.asarray(materialized[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]) - if values.shape != (len(person),): - raise ValueError( - f"{PRIVATE_SCHOOL_ATTENDANCE_OUTPUT} has {values.shape[0]} values for " - f"{len(person)} people." - ) - attends = values.astype(bool) + missing = [name for name in requested if name not in materialized] + if missing: + raise ValueError(f"Private-school attendance engine output missing: {missing}.") + restated = restate_private_school_attendance(frame, materialized, parameters) + if batched: + attends = restated + rank_route = PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE + else: + values = np.asarray(materialized[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]) + if values.shape != (len(person),): + raise ValueError( + f"{PRIVATE_SCHOOL_ATTENDANCE_OUTPUT} has {values.shape[0]} values " + f"for {len(person)} people." + ) + attends = values.astype(bool) + if not np.array_equal(attends, restated): + raise ValueError( + "policyengine-uk's attends_private_school, restated from the " + "engine's own values and parameters, differs from the engine's " + "flags over the same frame. The restatement no longer follows " + "the engine's formula, so a spine whose engine calls run clone by " + "clone would store other flags than one simulation over its pool." + ) + rank_route = PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE person[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT] = attends household_weights = frame.weights_for("household") weight_by_household = dict( @@ -176,7 +264,154 @@ def store_private_school_attendance( frame=result, year=int(year), weighted_pupils=float(person_weights[attends].sum()), + rank_route=rank_route, + engine_parity_checked=not batched, + ) + + +def restate_private_school_attendance( + frame: Frame, values: Mapping[str, Any], parameters: Any +) -> np.ndarray: + """policyengine-uk's ``attends_private_school`` over every person in ``frame``. + + ``values`` holds the engine's own arrays at one period for + :data:`PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS` and + :data:`PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS`: people in the frame's + order and households by ascending id, the frame's own order, as the + adapter returns them whether its call ran in one simulation or clone by + clone. ``parameters`` is the engine's parameter tree at that period, read + at :data:`PRIVATE_SCHOOL_ATTENDANCE_PARAMETERS` only. The steps, their + order and their types are the formula's (policyengine-uk 2.174.1, + variables/contrib/labour/attends_private_school.py), not its code: + + 1. each household's members' income tax plus National Insurance, summed + over the members in the people's order (policyengine-core's group sum, + a ``bincount``) and given to each member; + 2. net income: the household's market income plus its benefits, at each + member, less those taxes; + 3. each person carries the household's weight, so a household counts once + per member; while those weights sum below one million, nobody attends; + 4. each person's percentile of net income among the people of positive + weight, by microdf's weighted percentile rank, the rank the formula + calls; people of no weight keep percentile 0; + 5. the rate at that percentile, interpolated between the engine's rates at + every fifth percentile, times the engine's population factor, for + under-18s only; + 6. a person attends when the stored draw is below that rate. + + The adapter builds every simulation from data, so the formula's branch + for a situation, which imputes nobody, never applies. + """ + + person = frame.table("person") + household_ids = frame.table("household")["household_id"].to_numpy() + member_of = pd.Index(household_ids).get_indexer( + person["person_household_id"].to_numpy() + ) + if (member_of < 0).any(): + raise ValueError("A person's household is not in the frame's household table.") + arrays: dict[str, np.ndarray] = {} + for names, rows, entity in ( + (PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS, len(person), "person"), + (PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS, len(household_ids), "household"), + ): + for name in names: + if name not in values: + raise ValueError(f"The restatement needs the engine's {name!r}.") + array = np.asarray(values[name]) + if array.shape != (rows,): + raise ValueError( + f"The engine's {name!r} is not one value per {entity} of the frame." + ) + arrays[name] = array + taxes = np.bincount( + member_of, + weights=arrays["income_tax"] + arrays["national_insurance"], + minlength=len(household_ids), + )[member_of] + net_income = ( + arrays["household_market_income"][member_of] + + arrays["household_benefits"][member_of] + - taxes + ) + weights = arrays["household_weight"][member_of] + if weights.sum() < _IMPUTED_POPULATION_FLOOR: + return np.zeros(len(person), dtype=bool) + percentile = _net_income_percentiles(net_income, weights) + rate_path, factor_path = PRIVATE_SCHOOL_ATTENDANCE_PARAMETERS + rate = _parameter(parameters, rate_path) + rates = np.array([float(rate[str(p)]) for p in _RATE_PERCENTILES]) + rate_by_percentile = np.interp( + np.arange(_LOOKUP_PERCENTILES), list(_RATE_PERCENTILES), rates + ) + probability = ( + rate_by_percentile[percentile] + * _parameter(parameters, factor_path) + * arrays["age_under_18"] ) + return np.asarray(arrays[PRIVATE_SCHOOL_ATTENDANCE_DRAW] < probability, dtype=bool) + + +def _parameter(parameters: Any, path: str) -> Any: + """The node or value at a dotted ``path`` in a parameter tree.""" + + node = parameters + for name in path.split("."): + node = getattr(node, name) + return node + + +def _net_income_percentiles(net_income: np.ndarray, weights: np.ndarray) -> np.ndarray: + """Each person's percentile of net income among the people of positive + weight, in the frame's order: microdf's weighted percentile rank, the + rank policyengine-uk's formula calls, clipped to 0-100. People of no + weight keep 0, as the formula leaves them. microdf comes with + policyengine-uk, so it is imported only here.""" + + try: + from microdf import MicroSeries + except ImportError as exc: + raise ImportError( + "Restating private-school attendance ranks with microdf, which " + "policyengine-uk installs: install microcosm-build's 'uk' extra." + ) from exc + income = MicroSeries(net_income, weights=weights) + percentile = np.zeros(len(net_income), dtype=np.int64) + weighted = weights > 0 + percentile[weighted] = np.asarray( + income[weighted].percentile_rank().clip(0, 100) + ).astype(np.int64) + return percentile + + +def _engine_parameters(engine: object, year: int | str) -> Any: + """The engine's own parameter tree at ``year``, which the restatement + reads its rates and factor from.""" + + parameters = getattr(engine, "parameters", None) + if not callable(parameters): + raise TypeError( + "Private-school attendance restates policyengine-uk's formula from " + "the engine's own parameters, so its engine must give them " + "(parameters(period))." + ) + return parameters(year) + + +def _simulation_batches(engine: object, frame: Frame) -> int: + """How many simulations the engine computes ``frame`` in: one, or one per + clone (the UK adapter's ``simulation_batches``). An engine without the + method computes it in one.""" + + batches = getattr(engine, "simulation_batches", None) + if batches is None: + return 1 + count = batches(frame) + if isinstance(count, bool) or not isinstance(count, int | np.integer) or count < 1: + raise ValueError( + "The engine's simulation_batches must be a positive whole number." + ) + return int(count) def _assert_stage_parameters(stage: SourceStageSpec) -> None: diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_private_school_pooled_rank.py b/packages/microcosm-build/tests/engine/uk/test_uk_private_school_pooled_rank.py new file mode 100644 index 0000000000..f0bbfad1fc --- /dev/null +++ b/packages/microcosm-build/tests/engine/uk/test_uk_private_school_pooled_rank.py @@ -0,0 +1,517 @@ +"""Private-school attendance ranked over the pool (microcosm#931). + +policyengine-uk's ``attends_private_school`` ranks household net income over +the people a simulation holds. Once a dense spine's engine calls run clone by +clone, one clone's simulation would rank that clone alone, so stage 52 +restates the formula's last step over the whole frame from engine values each +household's own records decide, the engine's own parameters and microdf's +weighted percentile rank. + +The pool here is three clones of twelve made-up households: five alike in +every clone, whose incomes tie across clones, and the rest drawing their own +pay and weights, which spread about one to ten; clone 1 also holds a household +no other clone has. Most households have several children, members tie on +their household's income, and one household in each clone has no weight. +Household ids interleave the clones, benefit unit ids sort the other way and +the people are listed out of order. Each clone's people weigh less than the +formula's one million, so one clone's simulation imputes nobody, while the +pool's people weigh more. + +Each child's draw is set 3% under or 3% over the rate the formula gives that +child over the pool, read from the formula's own percentiles and the engine's +parameters, so a rank, a rate, its interpolation or the factor a few percent +off the formula's would store other flags. A spy on microdf's +``percentile_rank``, patched on the class, sees both the formula's call and +the restatement's. + +On it: the restatement hands microdf's rank the formula's net incomes and +weights and gets its percentiles back, bit for bit, in one simulation and +clone by clone, and its flags are the formula's; the stage run clone by clone +stores what one simulation over the pool stores; a population factor of zero, +or 6% lower, breaks the parity check; below the floor nobody attends on either +route, while design weights just under it impute on both, since the formula +reads the engine's weights at the calibration year; and a frame of one clone +stores what the stage stored before, the engine's attendance alone. +""" + +from __future__ import annotations + +import functools +from collections.abc import Mapping +from contextlib import contextmanager +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.private_school_attendance import ( + PRIVATE_SCHOOL_ATTENDANCE_DRAW, + PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE, + PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS, + PRIVATE_SCHOOL_ATTENDANCE_OUTPUT, + PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS, + PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE, + restate_private_school_attendance, + store_private_school_attendance, +) +from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine +from microcosm.frame.adapters.policyengine_uk_population import ( + UK_SPINE_ENGINE_BATCH_COLUMN, +) + +K = 3 +SOURCES = 12 +#: Sources whose records are the same in every clone. +ALIKE = 5 +#: The household with no weight, in every clone. +WEIGHTLESS = 8 +#: The household only clone 1 holds. +EXTRA = SOURCES +#: Each source's children, the household only clone 1 holds last. +CHILDREN = (0, 3, 2, 4, 1, 3, 4, 2, 3, 4, 0, 3, 2) +#: A household's weight before its own spread, about one to ten. +WEIGHT = 9_000.0 +D, P = 2024, 2025 +CLONE = UK_SPINE_ENGINE_BATCH_COLUMN +REGIONS = ("LONDON", "NORTH_WEST", "SCOTLAND", "WALES", "SOUTH_EAST") +INPUTS = ( + *PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS, + *PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS, +) +#: Where a child's draw sits against the rate the formula gives that child: +#: 3% under it (the child attends) or 3% over it (the child does not). +MARGINS = (0.97, 1.03) +#: Every child's draw until the draws are set: no percentile's rate admits it. +UNSET_DRAW = 0.5 +#: The draw of a child of no weight, which every positive percentile admits. +SMALL_DRAW = 0.001 +#: The people's design weights in all for the floor test: under the formula's +#: one million, while the engine's own weights at the calibration year, grown +#: from them, are over it. +NEAR_FLOOR = 0.9985e6 +#: The percentiles the engine gives its attendance rates at. +BREAKPOINTS = tuple(range(0, 101, 5)) + + +def _pool( + draws: Mapping[int, float] | None = None, + *, + scale: float = 1.0, + clone_column: bool = True, + one_clone: bool = False, + clones: tuple[int, ...] = tuple(range(K)), +): + """The made-up pool at the dataset year. ``draws`` gives each child's + draw by person id (without it every child draws UNSET_DRAW); ``scale`` + multiplies every weight; ``clone_column`` keeps the clone index on the + households and ``one_clone`` sets it to one clone for all; ``clones`` + keeps those clones' households only.""" + + persons, benunits, households, weights = [], [], [], [] + for clone in clones: + for source in (*range(SOURCES), *((EXTRA,) if clone == 1 else ())): + # Household ids interleave the clones; benefit unit ids sort the + # other way. + household_id = 10 * source + clone + 1 + benunit_id = 5_000 - 2 * household_id + alike = source < ALIKE + drawn = np.random.default_rng( + 7_000 + source + (0 if alike else 101 * clone) + ) + spread = float(10 ** drawn.uniform(-0.5, 0.5)) + weights.append(0.0 if source == WEIGHTLESS else WEIGHT * scale * spread) + households.append( + { + "household_id": household_id, + **({CLONE: 0 if one_clone else clone} if clone_column else {}), + "region": REGIONS[source % len(REGIONS)], + "council_tax": 1_400.0 + 100.0 * (source % 4), + "rent": 8_400.0 if source % 2 else 0.0, + "tenure_type": "RENT_PRIVATELY" if source % 2 else "OWNED_OUTRIGHT", + } + ) + adults, children = 1 + source % 2, CHILDREN[source] + benunits.append( + { + "benunit_id": benunit_id, + "dependent_children": children, + "is_married": adults == 2, + } + ) + for p in range(adults + children): + child = p >= adults + person_id = 10 * household_id + p + # Everyone takes a draw from the household's stream, so alike + # households stay alike; a child's own is replaced. + draw = float(drawn.uniform()) + if child: + draw = UNSET_DRAW if draws is None else draws[person_id] + # Some adults have no pay at all, so their households' incomes + # sit low; the others' pay is the source's, or the clone's own. + works = not child and (source + p) % 3 != 0 + persons.append( + { + "person_id": person_id, + "person_benunit_id": benunit_id, + "person_household_id": household_id, + "age": 3 + 3 * (p - adults) if child else 34 + 5 * p, + "is_benunit_head": p == 0, + "is_parent": children > 0 and not child, + "employment_income": ( + float(drawn.uniform(4_000, 90_000)) if works else 0.0 + ), + "self_employment_income": ( + float(drawn.uniform(2_000, 30_000)) + if p == 0 and source % 4 == 1 + else 0.0 + ), + # Stored at the root, as the spine stores it. + "months_since_last_birthday": float(drawn.integers(0, 12)), + PRIVATE_SCHOOL_ATTENDANCE_DRAW: draw, + } + ) + person = pd.DataFrame(persons).sample(frac=1.0, random_state=11) + household = pd.DataFrame(households) + order = np.argsort(household["household_id"].to_numpy(), kind="stable") + benunit = pd.DataFrame(benunits).sort_values("benunit_id") + return uk_national_frame( + person=person.reset_index(drop=True), + benunit=benunit.reset_index(drop=True), + household=household.iloc[order].reset_index(drop=True), + time_period=D, + household_weights=np.asarray(weights, dtype=float)[order], + ) + + +def _store(frame, engine=None): + return store_private_school_attendance( + frame, engine=engine or PolicyEngineUKEngine(), calibration_year=P + ) + + +def _stored(result) -> np.ndarray: + return result.frame.table("person")[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT].to_numpy() + + +def _member_of(frame) -> np.ndarray: + """Each person's household's row in the frame's household table.""" + + return pd.Index(frame.table("household")["household_id"]).get_indexer( + frame.table("person")["person_household_id"] + ) + + +def _person_weight_total(frame) -> float: + weights = np.asarray(frame.weights_for("household").values, dtype=float) + return float(weights[_member_of(frame)].sum()) + + +def _rate(parameters, percentile: np.ndarray) -> np.ndarray: + """The formula's attendance rate at each percentile: the engine's rates at + every fifth percentile, interpolated, times its population factor.""" + + vat = parameters.gov.simulation.private_school_vat + rates = [float(vat.private_school_attendance_rate[str(p)]) for p in BREAKPOINTS] + return np.interp(percentile, BREAKPOINTS, rates) * vat.private_school_factor + + +@contextmanager +def _ranks(): + """Every call of microdf's weighted percentile rank while open, patched on + the class so the formula's call and the restatement's are both seen: the + values ranked, their weights and the ranks returned.""" + + from microdf import MicroSeries + + calls: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = [] + rank = MicroSeries.percentile_rank + + @functools.wraps(rank) + def spy(self): + ranked = rank(self) + calls.append( + ( + pd.Series.to_numpy(self, copy=True), + self.weights.to_numpy(copy=True), + pd.Series.to_numpy(ranked, copy=True), + ) + ) + return ranked + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(MicroSeries, "percentile_rank", spy) + yield calls + + +def _assert_same_rank(seen, expected) -> None: + """One rank call is another, bit for bit: the same values and weights, in + the same order and types, and the same ranks back.""" + + for got, want in zip(seen, expected, strict=True): + assert got.dtype == want.dtype + np.testing.assert_array_equal(got, want) + + +@pytest.fixture(scope="module") +def draws(): + """Each child's draw, set 3% under or over the rate the formula gives it + over the pool, and who attends then. The rates come from the formula's own + percentiles, seen in its call of microdf's rank in one simulation over the + pool, and the engine's parameters at the calibration year; the draws + change nothing else the formula reads. A child of no weight, at + percentile 0 whose rate is zero, draws SMALL_DRAW.""" + + unset = _pool(clone_column=False) + engine = PolicyEngineUKEngine() + with _ranks() as calls: + values = engine.materialize( + unset, + [PRIVATE_SCHOOL_ATTENDANCE_OUTPUT, *INPUTS], + P, + dataset_period=str(D), + ) + # The formula ranks once. + ((_, _, ranked),) = calls + person_ids = unset.table("person")["person_id"].to_numpy() + weighted = np.asarray(values["household_weight"])[_member_of(unset)] > 0 + percentile = np.zeros(len(person_ids), dtype=np.int64) + percentile[weighted] = np.clip(ranked, 0, 100).astype(np.int64) + rate = _rate(engine.parameters(P), percentile) + children = np.asarray(values["age_under_18"], dtype=bool) + # By place in the household, so households alike in every clone keep + # alike draws. + margin = np.asarray(MARGINS)[person_ids % 10 % 2] + draw = np.where(rate > 0, rate * margin, SMALL_DRAW) + return SimpleNamespace( + by_person=dict(zip(person_ids[children], draw[children], strict=True)), + attends=children & (rate > 0) & (margin < 1), + ) + + +@pytest.fixture(scope="module") +def pool(draws): + """The pool without the clone index: one simulation over it.""" + + return _pool(draws.by_person, clone_column=False) + + +@pytest.fixture(scope="module") +def engine_values(pool): + """One simulation's attendance over the pool, beside the inputs the + restatement reads, at the calibration year from the dataset year, as the + stage asks for them; the engine's parameters there; and the formula's + call of microdf's rank.""" + + engine = PolicyEngineUKEngine() + with _ranks() as calls: + values = engine.materialize( + pool, + [PRIVATE_SCHOOL_ATTENDANCE_OUTPUT, *INPUTS], + P, + dataset_period=str(D), + ) + (rank,) = calls + return SimpleNamespace( + values=values, + parameters=engine.parameters(P), + rank=rank, + flags=np.asarray(values[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]).astype(bool), + ) + + +@pytest.fixture(scope="module") +def stored_single(pool): + """The stage over the pool in one simulation, and the rank calls made.""" + + with _ranks() as calls: + result = _store(pool) + return SimpleNamespace(result=result, ranks=calls) + + +def test_the_pool_is_under_the_floor_per_clone_and_over_it_in_all(): + for clone in range(K): + assert _person_weight_total(_pool(clones=(clone,))) < 1e6 + assert _person_weight_total(_pool()) > 1e6 + assert _person_weight_total(_pool(scale=0.25)) < 1e6 + + +def test_the_restatement_is_the_engines_formula_bit_for_bit(pool, engine_values, draws): + with _ranks() as calls: + restated = restate_private_school_attendance( + pool, engine_values.values, engine_values.parameters + ) + + # One call of microdf's rank, handed the formula's net incomes and + # weights in the formula's order and types, and returning its percentiles. + (rank,) = calls + _assert_same_rank(rank, engine_values.rank) + assert restated.dtype == bool + np.testing.assert_array_equal(restated, engine_values.flags) + # Every child sits where its draw was set: 3% under its rate it attends, + # 3% over it not, so these flags pin the rank, the rates, their + # interpolation and the factor to a few percent. + np.testing.assert_array_equal(engine_values.flags, draws.attends) + person = pool.table("person") + children = person["age"].to_numpy() < 18 + # Some children attend and some do not; no adult does. + assert engine_values.flags[children].any() + assert not engine_values.flags[children].all() + assert not engine_values.flags[~children].any() + # The weightless household's children keep percentile 0, whose rate is + # zero, so even a draw every positive percentile admits leaves them out. + source = (person["person_household_id"].to_numpy() - 1) // 10 + weightless = source == WEIGHTLESS + assert (weightless & children).any() + assert not engine_values.flags[weightless].any() + # The rank meets ties: households alike in every clone, and members of one + # household, share a net income. + weighted = ( + np.asarray(engine_values.values["household_weight"])[_member_of(pool)] > 0 + ) + net_income, _, _ = engine_values.rank + tied = pd.DataFrame({"net": net_income, "household": _member_of(pool)[weighted]}) + assert (tied.groupby("net")["household"].nunique() > 1).any() + assert tied.groupby("household")["net"].size().max() > 1 + + +def test_one_simulation_ranks_in_the_formula_then_the_restatement( + engine_values, stored_single +): + # The engine's formula ranks the pool, then the restatement ranks it + # again, from the same values and weights to the same percentiles. + formula, restatement = stored_single.ranks + _assert_same_rank(formula, engine_values.rank) + _assert_same_rank(restatement, engine_values.rank) + + +def test_the_stage_run_clone_by_clone_stores_what_one_simulation_stores( + draws, engine_values, stored_single +): + frame = _pool(draws.by_person) + engine = PolicyEngineUKEngine() + assert engine.simulation_batches(frame) == K + + with _ranks() as calls: + batched = _store(frame, engine) + + # No clone's simulation computes attendance, so the one rank is the + # restatement's over the pool: the formula's own, bit for bit. + (rank,) = calls + _assert_same_rank(rank, engine_values.rank) + single = stored_single.result + np.testing.assert_array_equal(_stored(single), engine_values.flags) + np.testing.assert_array_equal(_stored(batched), _stored(single)) + assert batched.rank_route == PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE + assert batched.engine_parity_checked is False + assert single.rank_route == PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE + assert single.engine_parity_checked is True + pupils = single.evidence()["weighted_pupils"] + assert batched.evidence()["weighted_pupils"] == pupils + # Why the stage restates: one clone's own simulation ranks that clone + # alone, and its people weigh under the floor, so it imputes nobody where + # the pool imputes some of the same people. + alone = _pool(draws.by_person, clone_column=False, clones=(1,)) + own = PolicyEngineUKEngine().materialize( + alone, [PRIVATE_SCHOOL_ATTENDANCE_OUTPUT], P, dataset_period=str(D) + ) + assert not np.asarray(own[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]).any() + in_clone = frame.table("household").set_index("household_id")[CLONE] + of_clone_1 = ( + frame.table("person")["person_household_id"].map(in_clone).to_numpy() == 1 + ) + assert _stored(batched)[of_clone_1].any() + + +class _PerturbedEngine(PolicyEngineUKEngine): + """The adapter, its parameter tree giving another population factor: a + tree the engine's own simulations do not hold.""" + + def __init__(self, factor): + super().__init__() + self._factor = factor + + def parameters(self, period): + node = super().parameters(period).gov.simulation.private_school_vat + perturbed = SimpleNamespace( + private_school_attendance_rate=node.private_school_attendance_rate, + private_school_factor=self._factor(node.private_school_factor), + ) + return SimpleNamespace( + gov=SimpleNamespace( + simulation=SimpleNamespace(private_school_vat=perturbed) + ) + ) + + +@pytest.mark.parametrize( + "factor", + [lambda factor: 0.0, lambda factor: 0.94 * factor], + ids=["no_factor", "factor_six_percent_lower"], +) +def test_a_perturbed_parameter_breaks_the_parity_check(pool, stored_single, factor): + # The engine's own flags have pupils, each 3% under its rate, so a + # restatement with no factor, or one 6% lower, differs from them. + assert _stored(stored_single.result).any() + + with pytest.raises(ValueError, match="differs from the engine's flags"): + _store(pool, _PerturbedEngine(factor)) + + +def test_below_the_floor_nobody_attends_on_either_route(draws): + clones = _pool(draws.by_person, scale=0.25) + engine = PolicyEngineUKEngine() + assert engine.simulation_batches(clones) == K + + batched = _store(clones, engine) + single = _store(_pool(draws.by_person, scale=0.25, clone_column=False)) + + assert batched.rank_route == PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE + assert single.rank_route == PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE + assert not _stored(batched).any() + assert not _stored(single).any() + + +def test_the_floor_reads_the_engines_weights_at_the_calibration_year(draws): + """The formula's one-million floor reads the engine's weights at the + calibration year, grown from the dataset year's design weights. Design + weights just under the floor are over it there, so both routes impute, + as one simulation's formula does.""" + + scale = NEAR_FLOOR / _person_weight_total(_pool()) + frame = _pool(draws.by_person, scale=scale, clone_column=False) + assert _person_weight_total(frame) < 1e6 + + single = _store(frame) + batched = _store(_pool(draws.by_person, scale=scale)) + + # The engine's own flags, stored once the restatement equalled them, have + # pupils, so its formula read weights over the floor; clone by clone, the + # restatement reads the same weights. + assert single.rank_route == PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE + assert _stored(single).any() + assert batched.rank_route == PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE + np.testing.assert_array_equal(_stored(batched), _stored(single)) + + +def test_a_frame_of_one_clone_stores_the_engines_flags_as_before(draws, engine_values): + frame = _pool(draws.by_person, one_clone=True) + engine = PolicyEngineUKEngine() + assert engine.simulation_batches(frame) == 1 + + result = _store(frame, engine) + + # What the stage asked the engine for before it restated anything, its + # attendance alone over the same frame, is what it stores; one simulation + # over the pool without the clone index gives the same. + before = PolicyEngineUKEngine().materialize( + frame, [PRIVATE_SCHOOL_ATTENDANCE_OUTPUT], P, dataset_period=str(D) + ) + np.testing.assert_array_equal( + _stored(result), + np.asarray(before[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]).astype(bool), + ) + np.testing.assert_array_equal(_stored(result), engine_values.flags) + assert result.rank_route == PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE + assert result.engine_parity_checked is True diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_private_school_attendance.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_private_school_attendance.py index 9c0777cef4..3982bbec89 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_private_school_attendance.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_private_school_attendance.py @@ -1,8 +1,17 @@ -"""Private-school attendance stored at the spine's weights (uk-data#568, microcosm#1095).""" +"""Private-school attendance stored at the spine's weights (uk-data#568, microcosm#1095). + +The stage restates policyengine-uk's formula over the frame from the engine's +own values and parameters (microcosm#931). The rank in it is microdf's, the +rank the formula calls, which the engine test holds to the formula bit for bit +(tests/engine/uk/test_uk_private_school_pooled_rank.py). Here a stand-in rank +records what the stage hands it, so the stage's arithmetic, both its routes, +its parity check and its floor run without the engine. +""" from __future__ import annotations from dataclasses import replace +from types import SimpleNamespace import numpy as np import pandas as pd @@ -10,23 +19,75 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.uk_runtime import private_school_attendance from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.private_school_attendance import ( PRIVATE_SCHOOL_ATTENDANCE_DRAW, + PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE, + PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS, PRIVATE_SCHOOL_ATTENDANCE_OUTPUT, + PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS, + PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE, UKPrivateSchoolAttendanceStageTransform, store_private_school_attendance, ) +INPUTS = ( + *PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS, + *PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS, +) +#: Household weights whose people (each household's weight once per member) +#: weigh above the formula's one million in all, and weights far under it. +WEIGHTS = (400_000.0, 300_000.0, 200_000.0) +LIGHT_WEIGHTS = (100.0, 50.0, 30.0) +#: The stand-in rank's percentiles for the four people. With the made-up rates +#: (a percentile's rate is the percentile over 100), the factor (0.5), the +#: under-18s (people 2 and 3) and the draws, only person 2 attends. +PERCENTILES = (80, 80, 30, 50) +ATTENDS = [False, True, False, False] + + +def _parameters(factor=0.5): + vat = SimpleNamespace( + private_school_attendance_rate={str(p): p / 100 for p in range(0, 101, 5)}, + private_school_factor=factor, + ) + return SimpleNamespace( + gov=SimpleNamespace(simulation=SimpleNamespace(private_school_vat=vat)) + ) + + +def _engine_values(weights): + """What the engine returns for the restatement: people in the frame's + order, households by id, in the engine's own types.""" + + return { + "income_tax": np.array([1_000.0, 0.0, 0.0, 200.0], dtype=np.float32), + "national_insurance": np.array([500.0, 0.0, 0.0, 0.0], dtype=np.float32), + "age_under_18": np.array([False, True, True, False]), + PRIVATE_SCHOOL_ATTENDANCE_DRAW: np.array( + [0.5, 0.01, 0.2, 0.9], dtype=np.float32 + ), + "household_market_income": np.array( + [30_000.0, 10_000.0, 15_000.0], dtype=np.float32 + ), + "household_benefits": np.array([2_000.0, 5_000.0, 1_000.0], dtype=np.float32), + "household_weight": np.array(weights, dtype=np.float32), + } + class _FakeEngine: - """The engine's attendance at the frame's weights, recorded per call.""" + """An engine that computes the frame in one simulation, with no + ``simulation_batches``: its attendance and the restatement's inputs, + recorded per call, and its parameters.""" country = "uk" - def __init__(self, attends): + def __init__(self, attends=ATTENDS, *, weights=WEIGHTS): self.attends = np.asarray(attends, dtype=bool) + self.values = _engine_values(weights) self.calls = [] + self.parameter_periods = [] def materialize(self, frame, variables, period, *, dataset_period=None): self.calls.append( @@ -37,10 +98,47 @@ def materialize(self, frame, variables, period, *, dataset_period=None): frame.weights_for("household").values.tolist(), ) ) - return {PRIVATE_SCHOOL_ATTENDANCE_OUTPUT: self.attends} + values = {**self.values, PRIVATE_SCHOOL_ATTENDANCE_OUTPUT: self.attends} + return {name: values[name] for name in variables} + + def parameters(self, period): + self.parameter_periods.append(period) + return _parameters() + + +class _BatchingEngine(_FakeEngine): + """An engine that says how many simulations it computes the frame in.""" + + def __init__(self, batches, attends=ATTENDS, **kwargs): + super().__init__(attends, **kwargs) + self.batches = batches + def simulation_batches(self, frame): + return self.batches -def _frame(**person_columns): + def materialize(self, frame, variables, period, *, dataset_period=None): + if self.batches > 1 and PRIVATE_SCHOOL_ATTENDANCE_OUTPUT in variables: + raise AssertionError("a call clone by clone refuses attendance") + return super().materialize( + frame, variables, period, dataset_period=dataset_period + ) + + +@pytest.fixture +def ranks(monkeypatch): + """A stand-in for microdf's rank that records what the stage hands it.""" + + seen = [] + + def stand_in(net_income, weights): + seen.append((net_income, weights)) + return np.asarray(PERCENTILES, dtype=np.int64) + + monkeypatch.setattr(private_school_attendance, "_net_income_percentiles", stand_in) + return seen + + +def _frame(weights=WEIGHTS, **person_columns): person = pd.DataFrame( { "person_id": [1, 2, 3, 4], @@ -55,7 +153,7 @@ def _frame(**person_columns): person=person, benunit=pd.DataFrame({"benunit_id": [10, 20, 30]}), household=pd.DataFrame({"household_id": [1, 2, 3], "region": ["LONDON"] * 3}), - household_weights=np.array([100.0, 50.0, 30.0]), + household_weights=np.array(weights), time_period="2024", ) @@ -64,34 +162,128 @@ def _stage(): return load_country_spec("uk").sources.stage_map()["private_school_attendance"] -def test_the_stage_stores_the_engines_attendance_at_the_spines_weights(): - engine = _FakeEngine([False, True, True, False]) +def _stored(result): + return result.frame.table("person")[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT] + + +def test_one_simulation_stores_the_engines_attendance_once_restated(ranks): + engine = _FakeEngine() result = store_private_school_attendance( _frame(), engine=engine, calibration_year=2025 ) - person = result.frame.table("person") - assert person[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT].tolist() == [ - False, - True, - True, - False, - ] - assert person[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT].dtype == bool + assert _stored(result).tolist() == ATTENDS + assert _stored(result).dtype == bool # One materialization at the calibration year, the frame's own period as - # the dataset year, at the spine's design weights. + # the dataset year, at the spine's design weights: the engine's attendance + # beside what the restatement reads, and the engine's parameters there. ((variables, period, dataset_period, weights),) = engine.calls - assert variables == (PRIVATE_SCHOOL_ATTENDANCE_OUTPUT,) + assert variables == (PRIVATE_SCHOOL_ATTENDANCE_OUTPUT, *INPUTS) assert (period, dataset_period) == (2025, "2024") - assert weights == [100.0, 50.0, 30.0] + assert weights == list(WEIGHTS) + assert engine.parameter_periods == [2025] + # The rank reads each person's household net income (market income plus + # benefits less the members' income tax and National Insurance) and the + # household's weight once per member, in the engine's types. + ((net_income, person_weights),) = ranks + assert net_income.dtype == np.float64 + assert net_income.tolist() == [30_500.0, 30_500.0, 15_000.0, 15_800.0] + assert person_weights.dtype == np.float32 + assert person_weights.tolist() == [400_000.0, 400_000.0, 300_000.0, 200_000.0] # The weights pass through and the receipt counts weighted pupils. assert result.frame.weights_for("household").values.tolist() == weights - assert result.weighted_pupils == 150.0 - assert result.evidence()["weights"] == "spine_design_weights" + assert result.weighted_pupils == 400_000.0 + evidence = result.evidence() + assert evidence["weights"] == "spine_design_weights" + assert evidence["rank_route"] == PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE + assert evidence["engine_parity_checked"] is True + + +def test_flags_the_restatement_does_not_reproduce_are_refused(ranks): + engine = _FakeEngine([False, True, True, False]) + + with pytest.raises(ValueError, match="differs from the engine's flags") as error: + store_private_school_attendance(_frame(), engine=engine, calibration_year=2025) + # It says which formula, never how many people differ. + assert not any(character.isdigit() for character in str(error.value)) + + +def test_a_frame_computed_clone_by_clone_stores_the_restatement(ranks): + engine = _BatchingEngine(3, attends=[True] * 4) + result = store_private_school_attendance( + _frame(), engine=engine, calibration_year=2025 + ) + + assert _stored(result).tolist() == ATTENDS + ((variables, period, dataset_period, _weights),) = engine.calls + assert variables == INPUTS + assert (period, dataset_period) == (2025, "2024") + evidence = result.evidence() + assert evidence["rank_route"] == PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE + assert evidence["engine_parity_checked"] is False + assert result.weighted_pupils == 400_000.0 + + +def test_one_batch_is_one_simulation(ranks): + engine = _BatchingEngine(1) + result = store_private_school_attendance( + _frame(), engine=engine, calibration_year=2025 + ) + + ((variables, *_),) = engine.calls + assert variables == (PRIVATE_SCHOOL_ATTENDANCE_OUTPUT, *INPUTS) + assert result.evidence()["rank_route"] == PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE + for batches in (0, True, 1.5): + with pytest.raises(ValueError, match="positive whole number"): + store_private_school_attendance( + _frame(), engine=_BatchingEngine(batches), calibration_year=2025 + ) -def test_the_stage_needs_the_stored_draw_and_writes_attendance_once(): - engine = _FakeEngine([False] * 4) +def test_under_one_million_people_of_weight_nobody_attends_on_either_route(ranks): + for engine in ( + _FakeEngine([False] * 4, weights=LIGHT_WEIGHTS), + _BatchingEngine(3, attends=[True] * 4, weights=LIGHT_WEIGHTS), + ): + result = store_private_school_attendance( + _frame(LIGHT_WEIGHTS), engine=engine, calibration_year=2025 + ) + assert not _stored(result).any() + # The floor comes before the rank, which never runs. + assert ranks == [] + with pytest.raises(ValueError, match="differs from the engine's flags"): + store_private_school_attendance( + _frame(LIGHT_WEIGHTS), + engine=_FakeEngine(ATTENDS, weights=LIGHT_WEIGHTS), + calibration_year=2025, + ) + + +def test_the_stage_needs_the_engines_parameters_and_its_inputs(ranks): + class _NoParameters(_FakeEngine): + parameters = None + + with pytest.raises(TypeError, match="parameters"): + store_private_school_attendance( + _frame(), engine=_NoParameters(), calibration_year=2025 + ) + + class _Partial(_FakeEngine): + def materialize(self, frame, variables, period, *, dataset_period=None): + values = super().materialize( + frame, variables, period, dataset_period=dataset_period + ) + values.pop("household_benefits") + return values + + with pytest.raises(ValueError, match="household_benefits"): + store_private_school_attendance( + _frame(), engine=_Partial(), calibration_year=2025 + ) + + +def test_the_stage_needs_the_stored_draw_and_writes_attendance_once(ranks): + engine = _FakeEngine() frame = _frame() no_draw = uk_national_frame( person=frame.table("person").drop(columns=[PRIVATE_SCHOOL_ATTENDANCE_DRAW]), @@ -105,6 +297,7 @@ def test_the_stage_needs_the_stored_draw_and_writes_attendance_once(): stored = _frame(**{PRIVATE_SCHOOL_ATTENDANCE_OUTPUT: [False] * 4}) with pytest.raises(ValueError, match="only writer"): store_private_school_attendance(stored, engine=engine, calibration_year=2025) + assert engine.calls == [] def test_only_the_isa_subscriptions_and_receipts_follow_the_stage(): @@ -128,7 +321,7 @@ def test_only_the_isa_subscriptions_and_receipts_follow_the_stage(): } -def test_the_stage_refuses_drifted_parameters(): +def test_the_stage_refuses_drifted_parameters(ranks): stage = _stage() operation, receipt = stage.operations drifted = replace( @@ -141,18 +334,14 @@ def test_the_stage_refuses_drifted_parameters(): ), ) transform = UKPrivateSchoolAttendanceStageTransform( - stage=drifted, engine=_FakeEngine([False] * 4), calibration_year=2025 + stage=drifted, engine=_FakeEngine(), calibration_year=2025 ) with pytest.raises(ValueError, match="drifted"): transform(_frame()) - stored = UKPrivateSchoolAttendanceStageTransform( - stage=stage, - engine=_FakeEngine([True, False, False, False]), - calibration_year=2025, - )(_frame()) - assert stored.table("person")[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT].tolist() == [ - True, - False, - False, - False, - ] + transform = UKPrivateSchoolAttendanceStageTransform( + stage=stage, engine=_FakeEngine(), calibration_year=2025 + ) + stored = transform(_frame()) + assert stored.table("person")[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT].tolist() == ATTENDS + evidence = transform.checkpoint_metadata()["evidence"] + assert evidence["rank_route"] == PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE From 48d12e52ea9ec5b8c0999b6a0c6a6153d50679da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:27:45 +0000 Subject: [PATCH 006/108] Describe corrected engine blocks in the dense runbook and the build-graph doc (#931) The dense runbook and the local pre-flight still said a per-clone engine run was exact when its blocks were identical copies scaled to the pool. Exact now means every block corrected to the pool's share denominator with a clean guard (exact_basis pool_share_injection_with_population_guard): equal to one engine over the pool to float32 rounding, not bitwise, with nothing computed in policyengine-uk's Universal Credit rebalancing years (2026 on). - docs/uk-dense-release-assembly-runbook-762.md says so where it describes --engine-blocks, the pre-flight and the engine_measurement limitation. It adds that a calibration year or dated measure's year of 2026 or later leaves every per-clone run inexact and refuses every batched spine call, and that the pre-flight does not require exact_basis, so a receipt from before the correction passes too; that is open for Max. - docs/uk-full-build-graph.md gains a section on the corrected measure blocks and their guard, on how a spine engine call over more than one clone runs clone by clone and what refuses it, and on stage 52's rank over the whole frame. - tools/preflight_uk_local_release_candidate.py: the docstring and the comment over the engine check only. The gate code is unchanged. test_uk_local_release_preflight gains two cases: a per-clone resolution whose representation has another mode, or another block count, fails however exact it says it is. - Two changelog fragments cover group (0). One describes the engine blocks, the spine batches, stage 52 and the shared modules. The other describes the annual-income factor for other spells, the one data change, which can move an annual input by one float32 step on both lines. Only comments, docs, fragments and tests change. No kernel hashes these files, so no node key, H2 fixture, source_stages.json or coverage manifest changes. Co-Authored-By: Claude Opus 5.5 --- ...nual-income-other-spells-growth.changed.md | 1 + ...31-uk-engine-blocks-pool-shares.changed.md | 6 ++ docs/uk-dense-release-assembly-runbook-762.md | 68 ++++++++++++------- docs/uk-full-build-graph.md | 8 +++ .../uk/test_uk_local_release_preflight.py | 16 ++++- tools/preflight_uk_local_release_candidate.py | 14 ++-- 6 files changed, 82 insertions(+), 31 deletions(-) create mode 100644 changelog.d/931-uk-annual-income-other-spells-growth.changed.md create mode 100644 changelog.d/931-uk-engine-blocks-pool-shares.changed.md diff --git a/changelog.d/931-uk-annual-income-other-spells-growth.changed.md b/changelog.d/931-uk-annual-income-other-spells-growth.changed.md new file mode 100644 index 0000000000..d051137ef9 --- /dev/null +++ b/changelog.d/931-uk-annual-income-other-spells-growth.changed.md @@ -0,0 +1 @@ +The annual-income simulation carries each input's other spells by policyengine-uk's own growth parameters (#931). The factor is the product, over each year after the dataset's up to the measurement year, of one plus the engine's rate in the series its dataset projection uses for the input: `gov.economic_assumptions.yoy_growth.obr.average_earnings` for pay and `gov.economic_assumptions.yoy_growth.obr.per_capita.mixed_income` for profit. It replaces the median of the engine's year-on-year ratios over the rows a simulation held, a statistic of whichever engine block ran, so every per-clone block and one engine over the pool now use the same factor. The engine's rows must still agree with it within 1e-5, or the resolver refuses. This moves data on the national and dense lines: an annual input with other spells can move by one float32 step, and with it the `*_annual` matrix entries, calibrated weights and H5 bytes. `other_spells_engine_factors` changes in every resolver receipt that carries it. diff --git a/changelog.d/931-uk-engine-blocks-pool-shares.changed.md b/changelog.d/931-uk-engine-blocks-pool-shares.changed.md new file mode 100644 index 0000000000..10539bea41 --- /dev/null +++ b/changelog.d/931-uk-engine-blocks-pool-shares.changed.md @@ -0,0 +1,6 @@ +Per-clone engine blocks now stand for the pool when the pool's clones differ (#931). In every simulation a dense measure block builds (`--engine-blocks K`), policyengine-uk's `shareholding`, `corporate_land_value` and `consumption_shareholding` are set, at each period the block computes, to the block's own value times `c_b = f_b·S_b/S`: its pool-mass factor times its allocation-key total over the pool's. The block first checks that the engine's own key total is `f_b·S_b` and that the engine carries each key and the weights by one factor per year. A guard then records every formula each simulation runs and judges policyengine-uk's whole-population formulas against a declaration: the share formulas corrected, `months_since_last_birthday` and `attends_private_school` stored, the engine's deciles and relative-poverty flags refused, and the rest zero under the parameters checked. Nothing a block or batch computes may fall in the Universal Credit rebalancing years (2026 on), where policyengine-uk draws new-claimant status by benefit-unit position. + +- **What exact means.** `engine_population_representation.exact` now holds only when every block applied its correction, passed its checks and got a clean verdict, and then means equal to one engine over the pool to float32 rounding, not bitwise. `exact_basis` (`pool_share_injection_with_population_guard`) names this basis, and `exact_scope`, which the release's `engine_measurement` limitation copies, is rewritten. A receipt without `exact_basis` predates the change, when exact meant identical copies. The measure step records a violation, which leaves the run not releasable. The release verdict, the local pre-flight, the dense assembler and the data contract read the same fields as before; none requires `exact_basis`. +- **Spine batches.** A spine engine call whose frame carries `household_clone_index` with more than one clone runs one simulation per clone, each reading the pool's shares (`c_b = S_b/S`) and judged under a pass that raises. A frame without the column, or with one clone, takes one simulation as before. Stage 52 (`private_school_attendance`) restates policyengine-uk's attendance over the whole frame from household-local engine values, with microdf's `MicroSeries.percentile_rank` and the engine's own parameters. Over one simulation it stores the engine's flags after checking the restatement bit for bit; clone by clone it stores the restatement. Its evidence gains `rank_route` and `engine_parity_checked`. +- **The shared pieces.** The country-agnostic machinery (pool shares, the checks, injection, the formula-run recorder, judging, `EngineBlock` and batches) is `microcosm.frame.adapters.engine_population`, and the UK declaration is `microcosm.frame.adapters.policyengine_uk_population`. Each block and batch first checks the declaration against the installed policyengine-uk; a single block and an unbatched spine call never read it. A re-pin that changes the whole-population list or a pinned formula source therefore leaves every dense block uncorrected and the run not releasable, and refuses every batched spine call, until the declaration is reviewed again. +- **What moves.** No spine frame carries `household_clone_index` yet, so every spine call takes one simulation and no spine data moves (frame, H5, stored flags), but stage 52's evidence gains its two fields on both lines, so its stage-evidence digest moves. Identical copies take `c_b` within float64 rounding of 1, so the correction leaves their measures bitwise as they were, and a single block, the national line's, gains none of the new receipt fields. Today's dense receipts gain `exact_basis`, `share_correction`, `population_reach` and `annual_income_projection_factors`; `block_sensitivity` now names engine variables, so `present_in_this_run` can list them, drops the SLC entry and gains `corrected_formulas`; and each block's resolver receipt gains `engine_pool_shares` and `engine_population_reach`. Every spine stage and the full and national target kernels re-key, and the engine-calling stages also bind the two new modules. The annual-income factor, the one data change, has its own fragment. diff --git a/docs/uk-dense-release-assembly-runbook-762.md b/docs/uk-dense-release-assembly-runbook-762.md index ed874baded..06d1dfe6ca 100644 --- a/docs/uk-dense-release-assembly-runbook-762.md +++ b/docs/uk-dense-release-assembly-runbook-762.md @@ -126,13 +126,26 @@ manifest, and refuses the national role's flags. `--release-candidate` pins the doctrine (bound 10, `nation_grain_family_equal_sqrt_count`, K=15, 2000 epochs). The rotated holdout runs only with `--rotated-holdout` (an opt-in, report-only diagnostic; `--skip-holdout` no longer exists, and the national -role refuses `--rotated-holdout`). The engine may resolve in a single block or per clone -(`--engine-blocks K`): each block's engine weights are scaled to the pool so -weight-share formulas see the pool's denominator, and the measures receipt -records whether the blocks were identical copies (an exact representation); -a per-block run is releasable only when it was. Best-effort staging -telemetry uploads every 300 s by default on this driver (the Hub allows about -128 commits per hour per repository). +role refuses `--rotated-holdout`). The engine may resolve in a single block or +per clone (`--engine-blocks K`). Per clone, each block's engine weights are +scaled to the pool's mass, and in every simulation the block builds, +policyengine-uk's share formulas (`shareholding`, `corporate_land_value`, +`consumption_shareholding`) are corrected to the pool's denominator. A guard +judges every whole-population formula the block ran against the UK +declaration. Nothing a block computes may fall in the Universal Credit +rebalancing years (2026 on), where policyengine-uk draws new-claimant status +by benefit-unit position (microcosm#931). The measures receipt records the +representation exact only when every block applied the correction, its checks +of the engine's key totals and growth held, and its guard verdict was clean; +`exact_basis` (`pool_share_injection_with_population_guard`) names that basis. +Exact then means equal to one engine over the pool to float32 rounding, not +bitwise. A receipt without `exact_basis` predates the correction, when exact +meant identical copies. A per-block run is releasable only when it is exact. +While policyengine-uk draws new-claimant status by position, a calibration +year, or a dated measure's year, of 2026 or later therefore leaves every +per-clone run inexact, and so not releasable, and refuses every batched spine +call. Best-effort staging telemetry uploads every 300 s by default on this +driver (the Hub allows about 128 commits per hour per repository). For scale: at K=25 with `--engine-blocks 25` the graph driver's dense solve took about 3 hours (2,000 epochs over 1.6 million households, M1 Max), each of three size-search probes about as long, and the build peaked at 24.7 GiB (the @@ -451,14 +464,18 @@ uv run --no-sync python tools/preflight_uk_local_release_candidate.py --candidat The pre-flight checks the manifest and the signed gate report for everything the contract will demand: release posture attested, shippable, every -release-blocking gate passed, an exact engine population (single block, or -per-block with identical copies scaled to the pool), the doctrine values (for -a sized run, the size doctrine, a realized count equal to the request and the -size search's L0 bracket), the -A15 census household uprating and A17 tenure application, the measure -exclusions and their windows, the holdout (recorded as not requested, or, when -`--rotated-holdout` asked for it, measured with five folds and a finite mean, -and in agreement with the request), the Logbook row, the artifact digest. +release-blocking gate passed, an exact engine population (a single block, or +one block per clone that the build recorded exact), the doctrine values (for a +sized run, the size doctrine, a realized count equal to the request and the +size search's L0 bracket), the A15 census household uprating and A17 tenure +application, the measure exclusions and their windows, the holdout (recorded +as not requested, or, when `--rotated-holdout` asked for it, measured with +five folds and a finite mean, and in agreement with the request), the Logbook +row, the artifact digest. A build now records one block per clone exact only +when every block was corrected to the pool's share denominator with a clean +guard. The pre-flight does not require `exact_basis`, so a receipt from before +the correction, when exact meant identical copies, passes too; whether it +should is open for Max's review. `--incumbent-manifest /incumbent_local_surface_manifest.json` checks that the extraction manifest you will attach exists. The pre-flight no @@ -509,15 +526,18 @@ uv run --no-sync python tools/evaluate_uk_incumbent_surface.py \ with `--incumbent-diagnostic` (or when `--steps` names them). The build's release-blocking fit gates read the measures it resolved, per -clone when `--engine-blocks K`; the build records that representation as exact -only for the weight-share scaling -(`engine_population_representation.exact_scope`), and engine draws that depend -on a record's position can differ between per-clone and single-block runs. The -release states this in its `engine_measurement` limitation; nothing re-measures -the released file on one engine unless this evaluation is run. Its assessment -marks any national or local row more than 25% from its target, or any family -of five or more rows with fewer than half within 25%. Signed deferrals and -measure exclusions do not exempt a row. +clone when `--engine-blocks K`. `engine_population_representation.exact_scope` +says what the representation's `exact` covers. When `exact` holds, the blocks +were corrected to the pool's share denominator with a clean guard, and nothing +they computed fell in the Universal Credit rebalancing years (2026 on), where +policyengine-uk draws new-claimant status by benefit-unit position. The +measures then equal one engine over the pool to float32 rounding, at the +measure step's starting weights. The release states this in its +`engine_measurement` limitation; nothing re-measures the released file on one +engine unless this evaluation is run. Its assessment marks any national or +local row more than 25% from its target, or any family of five or more rows +with fewer than half within 25%. Signed deferrals and measure exclusions do +not exempt a row. Use the actual metrics and weights filenames from the extraction manifest. Only one engine block is accepted. The evaluation records the pinned row diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index f8d0ff0ebe..8b7e22bf64 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -108,6 +108,14 @@ Shared machinery includes graph execution/storage/replay, typed artifacts, expli The full target compiler preserves the unreduced band-edge register, reference-period compilations, approved exclusions and frozen-register completeness checks before target selection. A nonempty country-only problem can have zero local rows. Local holdout is then inapplicable, while source, identity, mass and geographic integrity checks remain applicable. Omitted rows are never reported as fitted constraints. +## Engine blocks and spine batches + +`--engine-blocks K` resolves `uk.full.measures` with one policyengine-uk engine per clone rather than one over the pool; above one, it must equal `--n-clones`. Each block's household weights are scaled to the pool's mass. In every simulation the block builds (the measure simulation, its baseline, and the annual-income and counterfactual simulations with theirs), `shareholding`, `corporate_land_value` and `consumption_shareholding` are set, at each period the block computes, to the block's own value times `c_b = f_b·S_b/S`. That is the block's pool-mass factor times its allocation-key total over the pool's, both from totals the representation records (microcosm#931). Before it sets them, the block checks that the engine's own key total is `f_b·S_b` and that the engine carries each key and the weights by one factor per year. The annual-income simulation carries other spells by the engine's own growth parameters, so every block uses the same factor. A guard records every formula each simulation runs once the shares are set, and judges policyengine-uk's whole-population formulas against the UK declaration (`microcosm.frame.adapters.policyengine_uk_population`, on the shared `engine_population` machinery). The three share formulas must be the corrected inputs at every period, and `months_since_last_birthday` and `attends_private_school` stored inputs. The engine's income and wealth deciles and relative-poverty flags are refused, and any other such formula that runs must be zero under the parameters checked. Any work in the Universal Credit rebalancing years (2026 on) is refused too, since policyengine-uk draws new-claimant status there by benefit-unit position. The measure step records a violation rather than raising it. Once the blocks ran, `engine_population_representation.exact` holds only if every block applied its correction, passed its checks and got a clean verdict, and `exact_basis` (`pool_share_injection_with_population_guard`) names that basis. The measures then equal one engine over the pool to float32 rounding, not bitwise. An inexact per-block run is not releasable. So while policyengine-uk draws new-claimant status by position, a calibration year, or a dated measure's year, of 2026 or later leaves every per-clone run inexact, and so not releasable, and refuses every batched spine call. A single block, as on the national line, is exact by construction and records no correction or guard fields. + +A spine engine call whose frame carries more than one clone runs one simulation per clone, since one simulation over a cloned spine does not fit in memory. `PolicyEngineUKEngine.materialize` decides from the frame, not from a setting: a call whose household table carries `household_clone_index` with more than one clone runs its simulations in ascending clone order, and each is released before the next is built. A frame without the column, or with one clone, takes one simulation as before. A batched call's requests are judged first. A period in the rebalancing years, a refused formula, a frame without `months_since_last_birthday`, or a request for `attends_private_school`, `private_school_vat`, `baseline_private_school_vat` or `gov_tax` before stage 52 stores attendance refuses the call before any engine is built. Until then the spine's pass lets attendance run only where the three parameters the private-school VAT rate reads equal their baseline's and `private_school_vat_change` comes out zero, and only if `gov_tax` does not run. Unlike a measure block, each batch reads the pool's shares at the frame's own weights, unscaled (`c_b = S_b/S`), and is judged under the spine's pass, which raises on any violation rather than recording it, since the values come back as plain arrays. People come back in the frame's order and groups in ascending id order, as one simulation returns them. No spine frame carries `household_clone_index` yet, so every spine call takes one simulation today, and batching moves no spine value. + +policyengine-uk's `attends_private_school` ranks each household's market income plus benefits, less its income tax and National Insurance, over the people a simulation holds, so stage 52 (`private_school_attendance`) restates the formula's last step over the whole frame. It computes that income from household-local engine values, ranks it with microdf's `MicroSeries.percentile_rank`, and reads the attendance rates and population factor from the engine's own parameters. Over one simulation it also asks for the engine's flags, stores them once the restatement matches them bit for bit, and raises otherwise. Clone by clone it asks only for the household-local values and stores the restatement. Its evidence records `rank_route` (`engine_with_parity` or `pooled_restatement`) and `engine_parity_checked`. The spine sidecar's `rules_engine` gains `engine_batches` only on a spine whose calls run clone by clone, and the engine identity does not move with batching. Every stage that calls the engine through the adapter binds the adapter, `engine_population` and `policyengine_uk_population` into its implementation hash. + ## Replay and outputs The shared store defaults to `/.graph-store`. `--graph-store` can reuse another store. `--resume require` requires completed numerical nodes and evidence to be available; output materialization and byte readback still verify the recreated files. `--require-solve-reuse` requires only the solve, so a second release build can re-run the calibrated gates on it (see [a second build that reuses the solve](#a-second-build-that-reuses-the-solve)). `--resume-size-checkpoint` imports a legacy size-search checkpoint only after validating invocation identity, ordered target/household axes, initial weights and recomputed losses. It skips the saved dense solve and search. New runs persist their intermediates as graph artifacts before drawing. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py index d3ccc4e43c..029a654135 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_local_release_preflight.py @@ -459,12 +459,18 @@ def mutate(manifest): assert any(needle in failure for failure in failures), failures -def _per_clone(exact: bool = True, blocks: int = 15) -> dict: +def _per_clone( + exact: bool = True, + blocks: int = 15, + *, + mode: str = "block_weights_scaled_to_pool", + represented_blocks: int | None = None, +) -> dict: return { "blocks": blocks, "engine_population_representation": { - "mode": "block_weights_scaled_to_pool", - "blocks": blocks, + "mode": mode, + "blocks": blocks if represented_blocks is None else represented_blocks, "exact": exact, }, } @@ -478,6 +484,10 @@ def _per_clone(exact: bool = True, blocks: int = 15) -> dict: # One block per clone, so the count must be the run's K. (_per_clone(blocks=5), False), ({"blocks": 15}, False), + # A representation of another kind, or of another block count, is not + # this resolution's, however exact it says it is. + (_per_clone(mode="single_block"), False), + (_per_clone(represented_blocks=5), False), ], ) def test_one_engine_block_per_clone_passes_only_when_recorded_exact( diff --git a/tools/preflight_uk_local_release_candidate.py b/tools/preflight_uk_local_release_candidate.py index 2e75510cdf..d55f80d981 100644 --- a/tools/preflight_uk_local_release_candidate.py +++ b/tools/preflight_uk_local_release_candidate.py @@ -9,7 +9,7 @@ say what a publishable release candidate must say — release posture attested, shippable, every release-blocking gate passed, an exact engine population (one block, or one block per clone - whose identical copies are scaled to the pool), doctrine values + that the build recorded exact), doctrine values (and, for a sized run, the size doctrine: 150,000 households under the doctrine's draw, floor and refit penalty, selected by a size search in the doctrine's L0 bracket), the A15 census household @@ -291,9 +291,15 @@ def check_candidate_dir(candidate_dir: Path, *, today: date | None = None) -> li resolution = solve.get("measure_resolution", {}) blocks = resolution.get("blocks") representation = resolution.get("engine_population_representation") or {} - # One block per clone measures the pool exactly when the blocks are - # identical copies scaled to the pool (rowwise_cli.release_verdict); the - # build records that check, and nothing else stands in for it. + # One block per clone stands for the pool when the build recorded it exact + # (rowwise_cli.release_verdict): every block scaled to the pool's mass and + # corrected to its share denominator, with a clean guard, so the measures + # equal one engine over the pool to float32 rounding (exact_basis + # pool_share_injection_with_population_guard, microcosm#931). The build + # decides exact once its blocks ran, and nothing else stands in for it. + # The check does not require exact_basis, so a receipt from before the + # correction, when exact meant identical copies, passes too; whether it + # should is open for Max's review. per_clone_exact = ( type(blocks) is int and blocks > 1 From 369791d40e1d830a1271faf3bbd1766fbfa623e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:32:56 +0000 Subject: [PATCH 007/108] Read batch records from the frame and hold each batch's tree to the restatement (#931) Review follow-ups to batching spine engine calls by clone. - The spine sidecar's engine_batches is now read from the final frame, never from the calls this process ran. A resumed spine whose engine stages hit their checkpoints runs no kernel, so it under-reported batches; now it records what a fresh run does. PolicyEngineUKEngine.batch_record(frame) takes the same decision and refusals as simulation_batches. - PopulationDeclaration gains restated: stored formulas a caller may compute over the pool from household-local engine values, with the parameters each reads. Each pe-uk simulation builds and edits its own parameter tree, and parameters(period) is the adapter's fresh copy. So a new check_restated_parameters holds every batch's tree to the adapter's wherever a restatement reads it. The UK declares attends_private_school with its attendance rates and population factor, the parameters stage 52 reads. - PolicyEngineUKEngine(clones=...) accepts any whole number. A frame that drops the clone column still takes one simulation; G1's integration check rules that out after G1. - The test that finds the engine-calling stages matches a materialize call on any receiver, and any transform taking an engine. It keeps isa_subscriptions, which only checks the country, on an asserted allow-list. No data changes on today's lines: no spine frame carries household_clone_index before G1, so every call still takes one simulation and no sidecar gains engine_batches. Identities move again: every spine node re-keys through graph_kernels.py and the engine modules, and the full and national target kernels re-key through the two frame modules. The spine graph declaration (uk_spine.json), source_stages.json and the coverage manifest are unchanged. An engine test adds a frame carrying none of the keys' inputs, which gives the pool's values bitwise. A batch whose own tree moved a restated parameter is refused before it computes. Co-Authored-By: Claude Opus 5.5 --- .../build/uk_runtime/graph_kernels.py | 9 +- .../microcosm/build/uk_runtime/spine_build.py | 18 +- .../tests/engine_free/uk/test_uk_graph.py | 84 ++++++--- .../uk/test_uk_rules_engine_provenance.py | 109 +++++++++--- .../frame/adapters/engine_population.py | 160 +++++++++++++++--- .../frame/adapters/policyengine_uk.py | 79 +++++---- .../adapters/policyengine_uk_population.py | 16 +- .../engine/uk/test_policyengine_uk_batches.py | 121 +++++++++++-- .../uk/test_policyengine_uk_population.py | 13 ++ .../shared/test_engine_population.py | 128 +++++++++++++- .../uk/test_policyengine_uk_adapter.py | 42 +++-- 11 files changed, 649 insertions(+), 130 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index f74ee877bc..f52bd8f413 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -158,10 +158,11 @@ } # What every stage that calls the engine computes through: the adapter, which -# runs a call over more than one clone clone by clone, and the shared -# engine-block machinery with the UK declaration it judges each batch against -# (microcosm#931). An edit to any of them moves the hash of every such stage; -# test_uk_graph derives the stages from their engine calls. +# runs a call whose frame carries several clones one clone at a time, and the +# shared engine-block machinery with the UK declaration it judges each batch +# against (microcosm#931). An edit to any of them moves the hash of every such +# stage; test_uk_graph derives the stages from their engine calls and from +# their transforms' engine parameter. _ENGINE_MODULES = ( uk_engine_adapter, engine_population, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index 726d8699bc..ef1edfbce5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -761,16 +761,18 @@ def _rules_engine() -> PolicyEngineUKEngine: return PolicyEngineUKEngine() -def _rules_engine_record(engine: object) -> dict[str, object]: - """The sidecar's ``rules_engine``: the engine's provenance and, only once - one of its calls ran clone by clone, how they batched - (``engine_batches``, the adapter's :meth:`batch_record`, for the calls - this process ran). The engine identity is hashed from the provenance - alone, so it does not move with batching.""" +def _rules_engine_record(engine: object, frame) -> dict[str, object]: + """The sidecar's ``rules_engine``: the engine's provenance and, only when + the spine's final frame carries more than one clone, how the adapter + runs a call over it clone by clone (``engine_batches``, its + :meth:`batch_record`). Read from the final frame, never from the calls + this process ran: a resumed spine's stages that hit their checkpoints + run no call, and it records what a fresh run does. The engine identity is + hashed from the provenance alone, so it does not move with batching.""" record = _rules_engine_provenance() batch_record = getattr(engine, "batch_record", None) - batches = batch_record() if callable(batch_record) else None + batches = batch_record(frame) if callable(batch_record) else None if batches is not None: record["engine_batches"] = dict(batches) return record @@ -977,7 +979,7 @@ def _build_sidecar( "staging_delivery": dict(staging_delivery or {}), "spine_gate_report": spine_gate_report, "stochastic_contract_sha256": stochastic_contract_sha256, - "rules_engine": _rules_engine_record(engine), + "rules_engine": _rules_engine_record(engine, frame), # Which stages read the SPI Public Use Tape, whose Special Conditions # reach every output built on them (microcosm#1095). UK_SPI_PUBLIC_USE_TAPE_KEY: uk_spi_public_use_tape_marker(stages), diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 1db745f68b..2e76bfd799 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -335,10 +335,23 @@ def changed_helper_bytes(path: Path) -> bytes: assert registry.implementation_hash(control) == before[control] +#: What calls a rules engine, whatever it is called on: ``materialize`` (the +#: RulesEngine protocol), the shared batch driver, and the predictor +#: materialization of ``microcosm.frame.rules``, which calls it. +_ENGINE_CALLS = frozenset( + {"materialize", "materialize_in_batches", "materialize_rules_engine_predictors"} +) + +#: Stages whose transform takes the engine without computing through it, each +#: with why. The ISA subscriptions only check that the engine is the UK's +#: (assert_rules_engine_country). +_ENGINE_TAKEN_FOR_ITS_COUNTRY_ONLY = frozenset({"isa_subscriptions"}) + + def _calls_the_engine(module) -> bool: - """Whether a module calls a rules engine: ``engine.materialize`` or - ``self.engine.materialize``, or the predictor materialization of - ``microcosm.frame.rules``, which calls it.""" + """Whether a module calls a rules engine: any call named in + ``_ENGINE_CALLS``, on any receiver (``engine``, ``self.engine``, + ``self._engine``, ``rules_engine`` ...) or as a bare name.""" import ast import inspect @@ -347,14 +360,26 @@ def _calls_the_engine(module) -> bool: if not isinstance(node, ast.Call): continue function = node.func - if isinstance(function, ast.Attribute) and function.attr == "materialize": - target = function.value - if (isinstance(target, ast.Name) and target.id == "engine") or ( - isinstance(target, ast.Attribute) and target.attr == "engine" - ): - return True - name = getattr(function, "id", None) or getattr(function, "attr", None) - if name == "materialize_rules_engine_predictors": + if isinstance(function, ast.Attribute) and function.attr in _ENGINE_CALLS: + return True + if isinstance(function, ast.Name) and function.id in _ENGINE_CALLS: + return True + return False + + +def _takes_an_engine(module) -> bool: + """Whether a class the module defines (a stage transform) takes an + ``engine`` when it is built.""" + import inspect + + for _name, member in inspect.getmembers(module, inspect.isclass): + if member.__module__ != module.__name__: + continue + try: + parameters = inspect.signature(member).parameters + except (TypeError, ValueError): + continue + if "engine" in parameters: return True return False @@ -363,17 +388,19 @@ def test_every_stage_that_calls_the_engine_binds_the_engine_modules() -> None: """How an engine call runs, and clone by clone once a frame carries more than one clone, is the adapter's, the shared engine-block machinery's and the UK declaration's (microcosm#931). ``source_hash`` does not follow - imports, so every stage whose module calls the engine binds all three, - and no other stage binds any: one that only checks the engine's country - (isa_subscriptions) computes nothing through it.""" + imports, so every stage whose module calls the engine, or whose transform + takes one, binds all three, and no other stage binds any. The one + exception takes the engine only to check its country and computes + nothing through it.""" from microcosm.build.uk_runtime import graph_kernels engine_modules = set(graph_kernels._ENGINE_MODULES) - calling = { - stage + modules = { + stage: graph_kernels._stage_module(stage) for stage in graph_kernels._STAGE_MODULES - if _calls_the_engine(graph_kernels._stage_module(stage)) } + calling = {stage for stage, module in modules.items() if _calls_the_engine(module)} + taking = {stage for stage, module in modules.items() if _takes_an_engine(module)} assert { "frs_legacy_proxies", "frs_brma", @@ -382,15 +409,34 @@ def test_every_stage_that_calls_the_engine_binds_the_engine_modules() -> None: "private_school_attendance", "company_cars", } <= calling - assert "isa_subscriptions" not in calling + # The exception is still one: it takes the engine and never calls it. + assert _ENGINE_TAKEN_FOR_ITS_COUNTRY_ONLY <= taking - calling + computing = calling | (taking - _ENGINE_TAKEN_FOR_ITS_COUNTRY_ONLY) for stage in graph_kernels._STAGE_MODULES: bound = set(graph_kernels._STAGE_HELPER_MODULES.get(stage, ())) - if stage in calling: + if stage in computing: assert engine_modules <= bound, stage else: assert not engine_modules & bound, stage +def test_the_engine_call_scan_sees_any_receiver(tmp_path) -> None: + import types + + for source, calls in ( + ("rules_engine.materialize(frame, names, period)", True), + ("self._engine.materialize(frame, names, period)", True), + ("helpers.materialize_in_batches(frame, names, period)", True), + ("materialize_rules_engine_predictors(frame)", True), + ("frame.select(rows)", False), + ): + path = tmp_path / "stage.py" + path.write_text(f"def transform(self, frame):\n {source}\n") + module = types.ModuleType("stage") + module.__file__ = str(path) + assert _calls_the_engine(module) is calls, source + + @pytest.mark.parametrize( "module_name", ["policyengine_uk", "engine_population", "policyengine_uk_population"], diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py index b1f7da5d7c..5f8ae0570c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py @@ -10,6 +10,7 @@ import hashlib import json +from types import SimpleNamespace import pytest @@ -160,44 +161,56 @@ def missing(package: str) -> str: class _BatchingEngine: def __init__(self, record: dict | None) -> None: self.record = record + self.frames: list = [] - def batch_record(self) -> dict | None: + def batch_record(self, frame) -> dict | None: + self.frames.append(frame) return self.record _BATCHES = { "by": "household_clone_index", - "batched_calls": 3, - "batches": [15], + "batches": 15, "pass": "spine_batched", "declaration_sha256": "ab" * 32, } -def _sidecar(engine: object | None) -> dict: +def _final_frame(clones: tuple[int, ...] | None = None): + """A made-up final spine frame: one household per clone, or two + households without the clone index.""" + import pandas as pd from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.frame import WeightKind - frame = uk_national_frame( + households = 2 if clones is None else len(clones) + ids = list(range(1, households + 1)) + household = pd.DataFrame( + {"household_id": ids, "household_weight": [1.0 + i for i in ids]} + ) + if clones is not None: + household["household_clone_index"] = list(clones) + return uk_national_frame( person=pd.DataFrame( { - "person_id": [1, 2], - "person_benunit_id": [1, 2], - "person_household_id": [1, 2], - "age": [30.0, 40.0], + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": [30.0 + i for i in ids], } ), - benunit=pd.DataFrame({"benunit_id": [1, 2]}), - household=pd.DataFrame( - {"household_id": [1, 2], "household_weight": [1.0, 2.0]} - ), + benunit=pd.DataFrame({"benunit_id": ids}), + household=household, time_period=2023, weight_kind=WeightKind.DESIGN, ) + + +def _sidecar(engine: object | None, frame=None) -> dict: return spine_build._build_sidecar( - frame=frame, + frame=_final_frame() if frame is None else frame, stages=[], records=[], artifact_pins={}, @@ -211,24 +224,78 @@ def _sidecar(engine: object | None) -> dict: ) -def test_the_sidecar_records_batches_only_once_a_call_batched( +def test_the_sidecar_records_batches_only_for_a_final_frame_that_batches( monkeypatch: pytest.MonkeyPatch, ) -> None: # National never batches, and today no spine frame carries the clone - # index, so the sidecar's rules engine stays the provenance; a spine whose - # calls ran clone by clone adds how (microcosm#931). + # index, so the sidecar's rules engine stays the provenance; a spine + # whose final frame carries more than one clone adds how the adapter runs + # a call over it (microcosm#931). _install(monkeypatch, None) provenance = spine_build._rules_engine_provenance() + frame = _final_frame() for engine in (None, object(), _BatchingEngine(None)): - assert spine_build._rules_engine_record(engine) == provenance - assert _sidecar(engine)["rules_engine"] == provenance + assert spine_build._rules_engine_record(engine, frame) == provenance + assert _sidecar(engine, frame)["rules_engine"] == provenance batched = _BatchingEngine(dict(_BATCHES)) - assert spine_build._rules_engine_record(batched) == { + assert spine_build._rules_engine_record(batched, frame) == { **provenance, "engine_batches": _BATCHES, } - assert _sidecar(batched)["rules_engine"]["engine_batches"] == _BATCHES + assert _sidecar(batched, frame)["rules_engine"]["engine_batches"] == _BATCHES + # The record is asked of the final frame the sidecar describes. + assert batched.frames == [frame, frame] # The engine identity is hashed from the provenance alone. assert _identity(spine_build._rules_engine_provenance()) == _identity( {"package": "policyengine-uk", "version": VERSION} ) + + +def test_a_resumed_spine_records_the_batches_a_fresh_run_does( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A resumed spine's engine stages that hit their checkpoints run no call, + # so the record is read from the final frame, never from the calls this + # process ran: an adapter that ran a batched call and one that ran none + # (every engine stage a cache hit) write the same sidecar (microcosm#931). + import numpy as np + + from microcosm.frame.adapters import policyengine_uk as adapter_module + from microcosm.frame.adapters.engine_population import SPINE_BATCHED_PASS + from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine + from microcosm.frame.adapters.policyengine_uk_population import ( + UK_ENGINE_POPULATION, + UK_SPINE_ENGINE_BATCH_COLUMN, + ) + + _install(monkeypatch, None) + frame = _final_frame((0, 1, 2)) + monkeypatch.setattr( + adapter_module, + "materialize_in_batches", + lambda bundle, variables, period, **kwargs: { + name: np.zeros(bundle.n("person")) for name in variables + }, + ) + fresh, ran = PolicyEngineUKEngine(), PolicyEngineUKEngine() + for engine in (fresh, ran): + monkeypatch.setattr( + engine, + "_import_policyengine_uk", + lambda: SimpleNamespace(Microsimulation=None), + ) + ran.materialize(frame, ["age"], 2023) + expected = { + **spine_build._rules_engine_provenance(), + "engine_batches": { + "by": UK_SPINE_ENGINE_BATCH_COLUMN, + "batches": 3, + "pass": SPINE_BATCHED_PASS.name, + "declaration_sha256": UK_ENGINE_POPULATION.sha256, + }, + } + assert _sidecar(fresh, frame)["rules_engine"] == expected + assert _sidecar(ran, frame)["rules_engine"] == expected + # One clone, or no clone index (a national spine), records no batches. + for final in (_final_frame((0,)), _final_frame()): + assert "engine_batches" not in _sidecar(fresh, final)["rules_engine"] diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py index 53b824bab0..09cd04f80a 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_population.py @@ -59,8 +59,10 @@ forms each batch's share from the frame's own totals (:func:`frame_key_sums`, ``c_b = S_b / S`` at unscaled weights), runs an :class:`EngineBlock` on each batch's engine under a pass that raises, holds -the batches to one growth factor per key, and releases each engine before the -next is built. +each engine to the caller's parameters wherever a formula the caller +restates over the pool reads them (:func:`check_restated_parameters`) and +the batches to one growth factor per key, and releases each engine before +the next is built. Simulations are duck-typed policyengine-core simulations (``calculate``, ``set_input``, ``get_holder``, ``baseline``, ``branches``, ``branch_name``, @@ -120,6 +122,7 @@ "any_of", "assert_groups_nest_in_households", "check_key_sums", + "check_restated_parameters", "check_uniform_growth", "derived_watched_periods", "factor_spread", @@ -1013,9 +1016,13 @@ class PopulationDeclaration: the classes must cover exactly; ``observed`` names formulas recorded for receipts but not judged; ``constructor_factories`` names the structural reform factories the engine's constructor applies, which must each return - None; ``digests`` pins the sources the classification rests on - (:func:`source_sha256`). ``reviewed_version`` is the engine release the - classification was reviewed against; validation rests on the digests. + None; ``restated`` maps stored formulas a caller may compute over the + pool itself, from household-local engine values, to the parameter paths + that computation reads, which every batch's engine must hold as the + caller's tree does (:func:`check_restated_parameters`); ``digests`` pins + the sources the classification rests on (:func:`source_sha256`). + ``reviewed_version`` is the engine release the classification was + reviewed against; validation rests on the digests. """ engine: str @@ -1032,6 +1039,7 @@ class PopulationDeclaration: window: PositionWindow observed: tuple[str, ...] = () constructor_factories: tuple[str, ...] = () + restated: Mapping[str, tuple[str, ...]] = field(default_factory=dict) digests: Mapping[str, str] = field(default_factory=dict) def __post_init__(self) -> None: @@ -1045,6 +1053,13 @@ def __post_init__(self) -> None: ) object.__setattr__(self, "zero_at_policy", _frozen_mapping(self.zero_at_policy)) object.__setattr__(self, "inert", _frozen_mapping(self.inert)) + object.__setattr__( + self, + "restated", + _frozen_mapping( + {name: _names(paths) for name, paths in dict(self.restated).items()} + ), + ) object.__setattr__(self, "digests", _frozen_mapping(self.digests)) for name in ("stored", "refused", "observed", "constructor_factories"): object.__setattr__(self, name, _names(getattr(self, name))) @@ -1063,6 +1078,10 @@ def __post_init__(self) -> None: ) if not set(self.inert) <= set(self.stored): raise ValueError("An inert formula must also be declared stored.") + if not set(self.restated) <= set(self.stored): + raise ValueError("A restated formula must also be declared stored.") + if not all(self.restated.values()): + raise ValueError("A restated formula names the parameters it reads.") missing_keys = set(self.corrected.values()) - set(self.key_adds) if missing_keys: raise ValueError(f"Keys {sorted(missing_keys)} name no inputs.") @@ -1129,6 +1148,7 @@ def to_json(self) -> dict[str, Any]: "observed": sorted(self.observed), "window": self.window.to_json(), "constructor_factories": sorted(self.constructor_factories), + "restated": {name: list(paths) for name, paths in self.restated.items()}, "digests": dict(self.digests), } @@ -1189,8 +1209,9 @@ def validate_declaration(declaration: PopulationDeclaration, system: Any) -> Non every share formula is a formula over its key, every key a sum of plain inputs of the key's entity exactly as declared; the pinned sources are unchanged; the window's source starts at the window; every constructor - factory resolves; every predicate's parameters exist. Callers validate - when they first need the declaration, never at import. + factory resolves; every predicate's and every restatement's parameters + exist. Callers validate when they first need the declaration, never at + import. """ problems: list[str] = [] @@ -1283,8 +1304,14 @@ def validate_declaration(declaration: PopulationDeclaration, system: Any) -> Non continue if not callable(factory): problems.append(f"{reference} is not callable.") - for name, predicate in sorted(declaration.run_predicates().items()): - for path in predicate.paths(): + parameter_reads = { + name: predicate.paths() + for name, predicate in declaration.run_predicates().items() + } + for name, paths in declaration.restated.items(): + parameter_reads[name] = (*parameter_reads.get(name, ()), *paths) + for name, paths in sorted(parameter_reads.items()): + for path in paths: try: _parameter_value(system.parameters, path) except AttributeError: @@ -2436,6 +2463,70 @@ def _engine_rows(frame: Any, entity: str) -> np.ndarray: return frame.table(entity)[frame.schema.entity_id_column(entity)].to_numpy() +def _parameter_leaves(node: Any, path: str) -> dict[str, Any]: + """The values under a parameter at an instant, by path. A + policyengine-core node keeps its children in ``_children``; anything + else is one value.""" + + children = getattr(node, "_children", None) + if isinstance(children, Mapping): + leaves: dict[str, Any] = {} + for name in sorted(children, key=str): + leaves.update(_parameter_leaves(children[name], f"{path}.{name}")) + return leaves + return {path: node} + + +def _same_parameter(left: Any, right: Any, path: str) -> bool: + try: + ours = _parameter_leaves(_parameter_value(left, path), path) + theirs = _parameter_leaves(_parameter_value(right, path), path) + except AttributeError: + return False + if set(ours) != set(theirs): + return False + for leaf, value in ours.items(): + try: + if not np.array_equal(np.asarray(value), np.asarray(theirs[leaf])): + return False + except Exception: + # A value that cannot be compared is not known to be the same. + return False + return True + + +def check_restated_parameters( + simulation: Any, + reference: Callable[[Any], Any], + declaration: PopulationDeclaration, + period: Any, +) -> None: + """Refuse an engine whose parameters at ``period`` are not the caller's + wherever a restatement reads them (:attr:`PopulationDeclaration.restated`). + + A caller that restates a formula over the pool reads its parameters + from its own tree, ``reference`` (called with ``period``), while every + engine builds its own tree and may edit it while it is constructed. So + each batch's engine is held to the caller's, value by value under every + restated path, before it computes any value the call asks for. Raises + :class:`ValueError` naming the paths, never their values. + """ + + paths = sorted({path for paths in declaration.restated.values() for path in paths}) + if not paths: + return + own = simulation.tax_benefit_system.parameters(period) + expected = reference(period) + moved = [path for path in paths if not _same_parameter(own, expected, path)] + if moved: + raise ValueError( + f"A batch's engine holds other parameters at {moved} at {period} " + f"than the tree the restatement of {sorted(declaration.restated)} " + "over the pool reads, so that restatement would not be what the " + "engine computes." + ) + + def _run_batch( frame: Any, shares: Mapping[str, PoolShare], @@ -2446,10 +2537,13 @@ def _run_batch( declaration: PopulationDeclaration, policy: PassPolicy, build: Callable[[Any], Any], + reference_parameters: Callable[[Any], Any] | None, ) -> tuple[dict[str, np.ndarray], dict[str, float | None]]: - """One batch: its engine, an :class:`EngineBlock` on it, the requested - values and the block's verdict, and the engine released on the way out, - whether the batch passed or raised.""" + """One batch: its engine, an :class:`EngineBlock` on it (which validates + the declaration first), the engine held to the caller's parameters where + a restatement reads them, the requested values and the block's verdict, + and the engine released on the way out, whether the batch passed or + raised.""" simulation = build(frame) block = None @@ -2462,6 +2556,10 @@ def _run_batch( periods=(dataset_year, period), dataset_year=dataset_year, ) + if reference_parameters is not None: + check_restated_parameters( + simulation, reference_parameters, declaration, period + ) values: dict[str, np.ndarray] = {} for name in names: block.note_request(name, period) @@ -2515,24 +2613,30 @@ def materialize_in_batches( build: Callable[[Any], Any], entity_of: Callable[[str], str], household: str = "household", + reference_parameters: Callable[[Any], Any] | None = None, ) -> dict[str, np.ndarray]: """``variables`` at ``period`` over ``bundle``, one engine per batch. ``batches`` partition the bundle's households (:func:`household_batches`), ``build`` makes an engine from a batch's frame, ``entity_of`` names a variable's entity, and the bundle's values are ``dataset_year`` inputs - the engine projects to ``period``. Before any engine is built, the call's - requests are judged (a period in the window, a refused formula, a stored - formula the frame does not supply) and every batch's - :class:`PoolShare` is formed from the frame, ``c_b = S_b / S`` at its - own weights (:func:`frame_key_sums`). Then, batch by batch in the - batches' order: the batch's frame (``bundle.select``), its engine, an - :class:`EngineBlock` on it at ``dataset_year`` and ``period`` (checks - (a) and (b), the pool's shares injected, the recorder armed), the - requested values, the block's verdict, and the engine released before - the next is built. Every batch's engine must grow each key and the - weights by one factor (:data:`BATCH_FACTOR_AGREEMENT`), or the - dataset-year shares would not hold at ``period``. + the engine projects to ``period``. ``reference_parameters`` is the + caller's parameter tree at a period, which a declaration that restates + formulas over the pool (:attr:`PopulationDeclaration.restated`) + requires. Before any engine is built, the call's requests are judged (a + period in the window, a refused formula, a stored formula the frame does + not supply) and every batch's :class:`PoolShare` is formed from the + frame, ``c_b = S_b / S`` at its own weights (:func:`frame_key_sums`). + Then, batch by batch in the batches' order: the batch's frame + (``bundle.select``), its engine, an :class:`EngineBlock` on it at + ``dataset_year`` and ``period`` (checks (a) and (b), the pool's shares + injected, the recorder armed), the engine held to the caller's + parameters at ``period`` wherever a restatement reads them + (:func:`check_restated_parameters`), the requested values, the block's + verdict, and the engine released before the next is built. Every + batch's engine must grow each key and the weights by one factor + (:data:`BATCH_FACTOR_AGREEMENT`), or the dataset-year shares would not + hold at ``period``. The values come back as one engine over the bundle returns them: people in the bundle's order and groups by ascending id @@ -2556,6 +2660,13 @@ def materialize_in_batches( f"The dataset's year {dataset} is after {year}; the engine projects " "a dataset forward only." ) + if declaration.restated and reference_parameters is None: + raise ValueError( + f"The declaration restates {sorted(declaration.restated)} over the " + "pool from parameters every batch's engine must hold as the caller " + "does, so a batched call needs the caller's tree " + "(reference_parameters)." + ) names = tuple(dict.fromkeys(str(name) for name in variables)) entities = {name: str(entity_of(name)) for name in names} _judge_requests( @@ -2583,6 +2694,7 @@ def materialize_in_batches( declaration=declaration, policy=policy, build=build, + reference_parameters=reference_parameters, ) _refuse_unequal_growth(growth) for name in names: diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py index 04d36be3d0..bca07ab21f 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import operator import sys import threading from collections.abc import Iterator, Mapping, Sequence @@ -398,20 +399,28 @@ class PolicyEngineUKEngine: clone count when known, only refuses: a frame carrying the column with any other number of clones is part of a pool, or not this spine's, and an engine over it would read its policyengine-uk shares over the wrong - population. + population. A frame without the column still takes one simulation over + the rows it holds, as a stage before the clones exist must; that no + stage after them drops the column is the clone stage's integration + check, not this adapter's. """ country = "uk" def __init__(self, *, clones: int | None = None) -> None: - if clones is not None and ( - isinstance(clones, bool) or not isinstance(clones, int) or clones < 1 - ): - raise ValueError("clones is a positive whole number of clones, or None.") + if clones is not None: + # Any whole number (a numpy integer too), but not a boolean. + try: + count = None if isinstance(clones, bool) else operator.index(clones) + except TypeError: + count = None + if count is None or count < 1: + raise ValueError( + "clones is a positive whole number of clones, or None." + ) + clones = count self._system: Any = None self._clones = clones - # The batch count of each call that ran in batches (batch_record). - self._batch_counts: list[int] = [] def variable_metadata(self, name: str) -> VariableMetadata: """Return entity, dtype kind, and period semantics for a UK variable.""" @@ -466,10 +475,12 @@ def materialize( A bundle carrying more than one clone (:meth:`simulation_batches`) is computed clone by clone (:func:`materialize_in_batches`): each clone's - simulation reads the pool's shares (``c_b = S_b / S`` at the frame's - own weights), the UK declaration's guard judges what it ran under the - spine's pass, which raises on anything only the pool defines, and the - values come back as one simulation over the bundle returns them. + simulation must hold the parameters the UK declaration's + restatements read as :meth:`parameters` does, reads the pool's + shares (``c_b = S_b / S`` at the frame's own weights), the + declaration's guard judges what it ran under the spine's pass, which + raises on anything only the pool defines, and the values come back + as one simulation over the bundle returns them. """ microsimulation_class = self._import_policyengine_uk().Microsimulation @@ -518,31 +529,40 @@ def simulation_batches(self, bundle: Frame) -> int: return 1 if batches is None else len(batches) def parameters(self, period: int | str) -> Any: - """The engine's own parameters at ``period``. - - Read from the tax-benefit system every simulation this adapter builds - starts from, so a stage that restates an engine formula over the pool - takes the engine's parameters rather than a copy (microcosm#931). + """The engine's parameters at ``period``, from this adapter's own + fresh ``CountryTaxBenefitSystem``. + + No simulation reads this tree: each builds its own and edits it + while it is constructed. A call that runs in batches holds every + batch's tree equal to this one at the call's period wherever the UK + declaration's restatements read it (``UK_ENGINE_POPULATION.restated``: + the private-school attendance rates and population factor), and + raises otherwise, so a stage that restates the attendance formula + over the pool from these parameters reads what each batch's engine + holds (microcosm#931). """ return self._tax_benefit_system().parameters(period) - def batch_record(self) -> dict[str, object] | None: - """How this adapter's calls ran in batches, or None when none did. - - The column the batches follow, how many calls ran in batches, the - batch counts they ran in, and the pass and declaration every batch - was judged under (each was clean: a violation raises). Names and - counts of calls and clones only, never a record's value or a count - of records. + def batch_record(self, bundle: Frame) -> dict[str, object] | None: + """How a call over ``bundle`` runs in batches, or None when it takes + one simulation. + + The column the batches follow, the number of batches (one per + clone), and the pass and the declaration each batch is judged under + (a violation in any raises). Read from the frame alone, by the same + decision :meth:`materialize` makes, never from the calls this + process ran, so a spine resumed from its checkpoints records what a + fresh run of it does. Names and the clone count only, never a + record's value or a count of records. """ - if not self._batch_counts: + batches = self.simulation_batches(bundle) + if batches == 1: return None return { "by": UK_SPINE_ENGINE_BATCH_COLUMN, - "batched_calls": len(self._batch_counts), - "batches": sorted(set(self._batch_counts)), + "batches": batches, "pass": SPINE_BATCHED_PASS.name, "declaration_sha256": UK_ENGINE_POPULATION.sha256, } @@ -596,7 +616,7 @@ def build(frame: Frame) -> Any: validate_uc_claimant_input(simulation, frame.table("person"), period) return simulation - results = materialize_in_batches( + return materialize_in_batches( bundle, variables, int(period), @@ -606,9 +626,8 @@ def build(frame: Frame) -> Any: dataset_year=dataset_year, build=build, entity_of=lambda name: self.variable_metadata(name).entity, + reference_parameters=self.parameters, ) - self._batch_counts.append(len(batches)) - return results def export_contract(self) -> ExportContract: """UK export contracts are owned by microcosm-build for this release.""" diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk_population.py b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk_population.py index 86591b3efe..cb6b94ff57 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk_population.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk_population.py @@ -22,7 +22,12 @@ rate equals its baseline's; - refused: the household income and wealth deciles and the relative poverty flags rank or take a median over the simulated population, which no factor - corrects. + corrects; +- restated: attendance ranks household net income over the simulated + population, so a stage whose call runs in batches may compute it over the + pool from household-local engine values instead; every batch's engine must + then hold the attendance rates and the population factor the formula reads + as the adapter's own tree does. The window: policyengine-uk's constructor draws which Universal Credit claimants are new claimants by benefit-unit position for the rebalancing @@ -184,6 +189,15 @@ def _as_baseline(path: str) -> ParameterTest: "policyengine_uk.reforms.scotland.scottish_child_payment_reform:" "create_scottish_child_payment_reform", ), + # attends_private_school.py reads these two under + # gov.simulation.private_school_vat: the rates at the 21 percentile + # breakpoints and the population adjustment factor. + restated={ + "attends_private_school": ( + "gov.simulation.private_school_vat.private_school_attendance_rate", + "gov.simulation.private_school_vat.private_school_factor", + ), + }, digests={ "policyengine_uk/variables/household/wealth/shareholding.py": ( "95fae830757b8db478cab62c1e84b093af268abd8d0ed21a518c65b0d635a52b" diff --git a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_batches.py b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_batches.py index e3d38246f9..9ccbd06744 100644 --- a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_batches.py +++ b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_batches.py @@ -10,12 +10,16 @@ float32 bound, household taxes within its per-row form, household-local values bitwise, people in the frame's order and groups by id. What only the pool defines is refused before any simulation is built, and a frame with one -clone takes the single simulation it always has. Every household here is made -up. +clone takes the single simulation it always has. A frame that carries none of +the allocation keys' inputs (a call before the wealth and consumption stages +write them) gives the pool's values bitwise, its shares all zero, and a batch +whose own tree moved a parameter a restatement reads is refused. Every +household here is made up. """ from __future__ import annotations +import re import warnings from types import SimpleNamespace @@ -28,6 +32,7 @@ from microcosm.frame.adapters.engine_population import ( SPINE_BATCHED_PASS, PopulationGuardError, + check_restated_parameters, frame_key_sums, household_batches, pool_share_atol, @@ -74,6 +79,11 @@ "household_weight", ) REQUESTED = (*SHARES, "household_tax", *LOCAL) +#: The allocation keys' inputs, which no frame carries before the stages that +#: write wealth and consumption. +KEY_INPUTS = tuple( + column for adds in UK_ENGINE_POPULATION.key_adds.values() for column in adds +) def _tables(store): @@ -135,13 +145,15 @@ def _tables(store): ) -def _frame(*, clones=True, single=False, store=(MONTHS,)): +def _frame(*, clones=True, single=False, store=(MONTHS,), keys=True): person, benunit, household = _tables(store) weights = household.pop("weight").to_numpy() if single: household[UK_SPINE_ENGINE_BATCH_COLUMN] = 0 if not clones: household = household.drop(columns=[UK_SPINE_ENGINE_BATCH_COLUMN]) + if not keys: + household = household.drop(columns=list(KEY_INPUTS)) return Frame( tables={"person": person, "benunit": benunit, "household": household}, schema=UK_SCHEMA, @@ -289,12 +301,11 @@ def test_people_come_back_in_the_frames_order_and_groups_by_id(runs): assert not np.array_equal(people, np.sort(people)) -def test_the_call_ran_in_three_judged_batches(runs): +def test_the_frame_runs_in_three_judged_batches(runs): assert runs.engine.simulation_batches(runs.frame) == CLONES - assert runs.engine.batch_record() == { + assert runs.engine.batch_record(runs.frame) == { "by": UK_SPINE_ENGINE_BATCH_COLUMN, - "batched_calls": 1, - "batches": [CLONES], + "batches": CLONES, "pass": SPINE_BATCHED_PASS.name, "declaration_sha256": UK_ENGINE_POPULATION.sha256, } @@ -321,7 +332,41 @@ def refuse(*args, **kwargs): assert one[name].dtype == plain[name].dtype, name np.testing.assert_array_equal(one[name], plain[name], err_msg=name) assert engine.simulation_batches(_frame(single=True)) == 1 - assert engine.batch_record() is None + assert engine.batch_record(_frame(single=True)) is None + + +@pytest.mark.parametrize( + ("period", "dataset_period"), [(D, None), (P, D)], ids=["dataset_year", "projected"] +) +def test_a_frame_without_the_keys_inputs_gives_the_pools_values_bitwise( + period, dataset_period +): + # The earliest calls once the clones exist (the WAS stage's read before it + # writes wealth, the HBAI reads before the LCFS stage writes consumption) + # carry no allocation key input: every batch's total and the pool's are + # zero, so each batch takes zero shares, as the pool does. + names = (*REQUESTED, "household_net_income") + batched = _quiet( + PolicyEngineUKEngine().materialize, + _frame(keys=False), + names, + period, + dataset_period=dataset_period, + ) + single = _quiet( + PolicyEngineUKEngine().materialize, + _frame(clones=False, keys=False), + names, + period, + dataset_period=dataset_period, + ) + for name in names: + assert type(batched[name]) is type(single[name]), name + assert batched[name].dtype == single[name].dtype, name + np.testing.assert_array_equal(batched[name], single[name], err_msg=name) + for name in UK_ENGINE_POPULATION.corrected: + assert not np.any(single[name]), name + assert np.any(single["income_tax"] != 0) def test_stored_attendance_may_be_asked_for_in_batches(): @@ -399,7 +444,6 @@ def test_what_only_the_pool_defines_is_refused_before_any_simulation( engine.materialize(_frame(store=store), names, period, dataset_period=D) found = {(v.kind, v.name, v.period) for v in raised.value.verdict.violations} assert found == expected - assert engine.batch_record() is None def test_a_frame_with_fewer_clones_than_the_spine_is_refused(monkeypatch, system): @@ -410,8 +454,65 @@ def test_a_frame_with_fewer_clones_than_the_spine_is_refused(monkeypatch, system engine.simulation_batches(_frame(single=True)) -def test_the_adapters_parameters_are_the_engines(system): +def test_the_adapters_parameters_are_each_simulations_where_restated(system): + from policyengine_uk import Microsimulation + engine = PolicyEngineUKEngine() node = engine.parameters(P).gov.simulation.private_school_vat expected = system.parameters(P).gov.simulation.private_school_vat assert node.private_school_factor == expected.private_school_factor + # A simulation builds and edits its own tree; where a restatement reads + # it, it still holds the adapter's. + simulation = _quiet( + Microsimulation, dataset=engine._build_dataset(_frame(clones=False), D) + ) + assert simulation.tax_benefit_system is not engine._tax_benefit_system() + check_restated_parameters(simulation, engine.parameters, UK_ENGINE_POPULATION, P) + + +#: For each parameter path a restatement reads, a value under it (a child, or +#: the parameter itself) and where a batch engine's own edit moves it. +MOVES = { + "gov.simulation.private_school_vat.private_school_attendance_rate": ("50", 0.5), + "gov.simulation.private_school_vat.private_school_factor": (None, 0.5), +} + + +def test_every_restated_parameter_has_a_move(): + restated = UK_ENGINE_POPULATION.restated.values() + assert set(MOVES) == {path for paths in restated for path in paths} + + +@pytest.mark.parametrize("path", sorted(MOVES)) +def test_a_batch_whose_own_tree_moved_a_restated_parameter_is_refused( + monkeypatch, system, path +): + from policyengine_uk import Microsimulation + + child, value = MOVES[path] + built = [] + + class Moved(Microsimulation): + """policyengine-uk's simulation, editing its own tree once built.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + node = self.tax_benefit_system.parameters + for part in path.split("."): + node = getattr(node, part) + (node if child is None else node.children[child]).update(value=value) + self.tax_benefit_system.reset_parameter_caches() + built.append(self) + + engine = PolicyEngineUKEngine() + engine._system = system + monkeypatch.setattr( + engine, + "_import_policyengine_uk", + lambda: SimpleNamespace(Microsimulation=Moved), + ) + with pytest.raises(ValueError, match=re.escape(f"['{path}'] at {P}")): + _quiet(engine.materialize, _frame(), ["income_tax"], P, dataset_period=D) + # The first batch's engine is refused before it computes what was asked. + (simulation,) = built + assert not simulation.get_holder("income_tax").get_known_periods() diff --git a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_population.py b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_population.py index 0fb009de66..52c872dabb 100644 --- a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_population.py +++ b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_population.py @@ -187,6 +187,19 @@ def test_the_inert_predicate_gives_the_engines_rate_equality(system, year): } +def test_the_restated_parameters_are_all_the_attendance_formula_reads(system): + # A stage whose call runs in batches may restate attendance over the pool + # from the parameters the declaration names, which every batch's engine + # is held to (microcosm#931), so they are every parameter the engine's + # formula reads. + source = inspect.getsource(type(system.variables["attends_private_school"])) + assert source.count("parameters(period)") == 1 + ((name, base),) = re.findall(r"(\w+) = parameters\(period\)\.([a-z_.]+)", source) + read = {f"{base}.{leaf}" for leaf in re.findall(rf"\b{name}\.([a-z_]+)", source)} + assert read == set(UK_ENGINE_POPULATION.restated["attends_private_school"]) + assert set(UK_ENGINE_POPULATION.restated) == {"attends_private_school"} + + _LSR = "gov.simulation.labour_supply_responses" _EMPLOYER_NI = "gov.contrib.policyengine.employer_ni" _BUDGET = "gov.contrib.policyengine.budget" diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py b/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py index f45acf2487..2099d4ee21 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_engine_population.py @@ -41,6 +41,7 @@ any_of, assert_groups_nest_in_households, check_key_sums, + check_restated_parameters, check_uniform_growth, derived_watched_periods, factor_spread, @@ -1308,6 +1309,12 @@ def test_pool_shares_are_one_blocks_for_the_declared_formulas(): ({"stored": ("months",)}, "inert formula must also be declared stored"), ({"corrected": {"share": "other_key"}}, "name no inputs"), ({"refused": ("decile", "decile")}, "repeat"), + ( + {"restated": {"decile": ("gov.vat_rate",)}}, + "restated formula must also be declared stored", + ), + ({"restated": {"attendance": ()}}, "names the parameters it reads"), + ({"restated": {"attendance": ("gov.vat_rate",) * 2}}, "repeat"), ], ) def test_a_declaration_refuses_overlapping_or_incomplete_classes(changes, message): @@ -1318,6 +1325,9 @@ def test_a_declaration_refuses_overlapping_or_incomplete_classes(changes, messag def test_the_declaration_digest_follows_its_content(): assert _declaration().sha256 == _declaration().sha256 assert _declaration().sha256 != _declaration(observed=()).sha256 + restating = _declaration(restated={"attendance": ("gov.vat_rate",)}) + assert restating.sha256 != _declaration().sha256 + assert restating.to_json()["restated"] == {"attendance": ["gov.vat_rate"]} assert _declaration().classified() == { "share", "months", @@ -1400,6 +1410,7 @@ def test_validation_fails_closed_on_every_kind_of_drift(fake_engine, pinned_pack digests={"fake_pinned_engine/share.py": digest}, constructor_factories=(f"{FAKE_ENGINE}:WINDOW",), zero_at_policy={"budget_change": ParameterTest("gov.missing", "eq", 0)}, + restated={"attendance": ("gov.vat_rate", "gov.no_rate")}, ) fake_engine.WHOLE_POPULATION = fake_engine.WHOLE_POPULATION | {"extra"} fake_engine.WINDOW = range(2027, 2033) @@ -1416,8 +1427,10 @@ def test_validation_fails_closed_on_every_kind_of_drift(fake_engine, pinned_pack "no longer starts the window at 2026", "is not callable", "no parameter gov.missing", + "attendance: the engine has no parameter gov.no_rate", ): assert fragment in message, message + assert "gov.vat_rate" not in message def test_validation_requires_formulas_over_plain_inputs(fake_engine): @@ -1788,17 +1801,27 @@ def _engine_of(frame, *, system=None, growth=GROWTH, key_scale=1.0): ) -def _in_batches(frame, names, *, period=P, build=None, policy=SPINE_BATCHED_PASS): +def _in_batches( + frame, + names, + *, + period=P, + build=None, + policy=SPINE_BATCHED_PASS, + declaration=None, + reference=None, +): return materialize_in_batches( frame, names, period, batches=household_batches(frame, "clone"), - declaration=_declaration(), + declaration=_declaration() if declaration is None else declaration, policy=policy, dataset_year=D, build=_engine_of if build is None else build, entity_of=lambda name: "household", + reference_parameters=reference, ) @@ -1992,3 +2015,104 @@ def test_a_call_in_batches_needs_a_raising_pass_two_batches_and_no_back_cast( entity_of=lambda name: "household", ) gc.collect() + + +# --------------------------------------------------------------------------- +# Restated formulas: each batch's engine holds the caller's parameters +# --------------------------------------------------------------------------- + +#: The fake attendance restated over the pool from the VAT rate. +_RESTATING = {"restated": {"attendance": ("gov.vat_rate",)}} + + +def test_a_call_that_restates_needs_the_callers_tree(fake_engine): + built = [] + + def build(batch_frame): + built.append(batch_frame) + return _engine_of(batch_frame) + + with pytest.raises(ValueError, match="needs the caller's tree"): + _in_batches( + _batched_frame(), + ["local"], + build=build, + declaration=_declaration(**_RESTATING), + ) + assert built == [] + + +def test_each_batch_is_held_to_the_callers_parameters_before_it_computes( + fake_engine, +): + frame = _batched_frame() + declaration = _declaration(**_RESTATING) + values = _in_batches( + frame, + ["local"], + declaration=declaration, + reference=_fake_system().parameters, + ) + np.testing.assert_array_equal( + values["local"], _engine_of(frame).calculate("local", P) + ) + # The first batch's engine holds another rate than the caller's tree: the + # call stops there, before that engine computes any value asked for. + built = [] + + def build(batch_frame): + simulation = _engine_of(batch_frame) + built.append(simulation.holders["local"]) + return simulation + + with pytest.raises(ValueError, match=r"\['gov.vat_rate'\] at 2025") as raised: + _in_batches( + frame, + ["local"], + build=build, + declaration=declaration, + reference=_fake_system(vat_rate=0.25).parameters, + ) + assert numbers_are_years([str(raised.value)]) + (holder,) = built + assert holder.arrays == {} + # A declaration that restates nothing reads no parameters at all. + check_restated_parameters(object(), None, _declaration(), P) + + +class _CoreNode: + """A parameter node at an instant as policyengine-core holds one: its + children in ``_children``, each also an attribute.""" + + def __init__(self, **children): + self._children = children + for name, child in children.items(): + setattr(self, name, child) + + +def _school_tree(*, rate=0.025, extra=False, factor=0.85): + rates = {"0": 0.0, "5": rate, **({"10": 0.01} if extra else {})} + return _CoreNode( + gov=_CoreNode(school=_CoreNode(rates=_CoreNode(**rates), factor=factor)) + ) + + +def test_restated_parameters_are_compared_value_by_value_under_a_node(): + declaration = _declaration( + restated={"attendance": ("gov.school.rates", "gov.school.factor")} + ) + simulation = SimpleNamespace( + tax_benefit_system=SimpleNamespace(parameters=lambda period: _school_tree()) + ) + check_restated_parameters(simulation, lambda period: _school_tree(), declaration, P) + for moved, paths in ( + (_school_tree(rate=0.03), r"\['gov.school.rates'\]"), + (_school_tree(extra=True), r"\['gov.school.rates'\]"), + (_school_tree(factor=0.8), r"\['gov.school.factor'\]"), + (_CoreNode(gov=_CoreNode()), r"\['gov.school.factor', 'gov.school.rates'\]"), + ): + with pytest.raises(ValueError, match=paths) as raised: + check_restated_parameters( + simulation, lambda period, tree=moved: tree, declaration, P + ) + assert numbers_are_years([str(raised.value)]) diff --git a/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py b/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py index 3bc2574447..368b80eb0d 100644 --- a/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py +++ b/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py @@ -559,13 +559,25 @@ def refuse(): monkeypatch.setattr(engine, "_import_policyengine_uk", refuse) +def _record(batches): + return { + "by": UK_SPINE_ENGINE_BATCH_COLUMN, + "batches": batches, + "pass": SPINE_BATCHED_PASS.name, + "declaration_sha256": UK_ENGINE_POPULATION.sha256, + } + + def test_whether_a_call_batches_is_decided_from_the_frame(monkeypatch): engine = PolicyEngineUKEngine() _no_engine(monkeypatch, engine) assert engine.simulation_batches(_clone_frame(column=False)) == 1 assert engine.simulation_batches(_clone_frame((0,))) == 1 assert engine.simulation_batches(_clone_frame((0, 1, 2))) == 3 - assert engine.batch_record() is None + # The record is the frame's, with no call run: names and the clone count. + assert engine.batch_record(_clone_frame(column=False)) is None + assert engine.batch_record(_clone_frame((0,))) is None + assert engine.batch_record(_clone_frame((0, 1, 2))) == _record(3) @pytest.mark.parametrize( @@ -601,12 +613,21 @@ def test_the_spines_clone_count_refuses_any_other_frame( engine.materialize(frame, ["age"], 2025) -@pytest.mark.parametrize("clones", [0, -1, True, 2.0, "3"]) +@pytest.mark.parametrize("clones", [0, -1, True, np.bool_(True), 2.0, "3"]) def test_the_spines_clone_count_is_a_positive_whole_number(clones): with pytest.raises(ValueError, match="positive whole number"): PolicyEngineUKEngine(clones=clones) +@pytest.mark.parametrize("clones", [3, np.int64(3), np.int32(3)]) +def test_the_spines_clone_count_may_be_any_whole_number_type(monkeypatch, clones): + engine = PolicyEngineUKEngine(clones=clones) + _no_engine(monkeypatch, engine) + assert engine.simulation_batches(_clone_frame((0, 1, 2))) == 3 + with pytest.raises(ValueError, match="no engine call runs per clone"): + engine.simulation_batches(_clone_frame((0, 1))) + + def test_a_clone_index_with_missing_households_is_refused(monkeypatch): engine = PolicyEngineUKEngine() _no_engine(monkeypatch, engine) @@ -667,7 +688,7 @@ def refuse(*args, **kwargs): values = engine.materialize(frame, ["age"], 2025) assert len(built) == 1 and len(built[0]) == frame.n("person") np.testing.assert_array_equal(values["age"], np.arange(frame.n("person"))) - assert engine.batch_record() is None + assert engine.batch_record(frame) is None def test_a_frame_with_clones_runs_through_the_shared_batches(monkeypatch): @@ -694,22 +715,21 @@ def batched(bundle, variables, period, **kwargs): assert kwargs["dataset_year"] == 2023 assert [batch.key for batch in kwargs["batches"]] == [0, 1, 2] assert kwargs["entity_of"]("age") == "person" + # Every batch is held to the adapter's own parameters wherever the UK + # declaration's restatements read them. + assert kwargs["reference_parameters"] == engine.parameters # Each batch's engine is built from the batch's own frame, at the # dataset's year, and its roles are checked against the batch's people. batch = frame.select(kwargs["batches"][1].persons) kwargs["build"](batch) assert built == [list(range(batch.n("person")))] assert roles == [(batch.n("person"), 2025)] - assert engine.batch_record() == { - "by": UK_SPINE_ENGINE_BATCH_COLUMN, - "batched_calls": 1, - "batches": [3], - "pass": SPINE_BATCHED_PASS.name, - "declaration_sha256": UK_ENGINE_POPULATION.sha256, - } engine.materialize(frame, ["age"], 2025) assert calls[-1][3]["dataset_year"] == 2025 - assert engine.batch_record()["batched_calls"] == 2 + # The calls this adapter ran leave no trace in the record: it is the + # frame's, as a fresh adapter reads it. + assert engine.batch_record(frame) == _record(3) + assert PolicyEngineUKEngine().batch_record(frame) == _record(3) with pytest.raises(ValueError, match="projects a dataset forward only"): engine.materialize(frame, ["age"], 2023, dataset_period=2025) From 3229048673896b86fed43f0a0f8b6b08a3cec887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:25:17 +0000 Subject: [PATCH 008/108] Read the survey's net income in both WAS stages (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WAS donor's income, DVTotinc_bhcR8, is net regular household income before housing costs less council tax: HBAI's concept. Both WAS-trained stages matched it to the engine's household_net_income, which also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds benefits in kind. At was_lisa (stage 24) those shares already read the corporate wealth was_wealth had just drawn, at the spine's weights before calibration. Both stages now read the engine's HBAI net income before the reset to zero, the concept LCFS, ETB services and NTS read (#1164), named hbai_household_net_income on donor and recipient (María's ruling, 6 Nov). - was_wealth materializes UK_HBAI_NET_INCOME_VARIABLE, and UK_WAS_ENGINE_PREDICTOR_NAMES gives it the predictor name UK_HBAI_NET_INCOME_PREDICTOR; the donor cleaning maps DVTotinc_bhcR8 to the same name. The chain's segments, their order and seeds are unchanged. - Its derive step declares the donor column in direct_maps, and the drift check refuses a declared direct map the cleaning does not read. - was_lisa takes the WAS household predictor, so the ownership logit, the balance forest and the realised profile's income tertiles read it. The profile key becomes by_hbai_household_net_income_tertile and the logit's feature log1p_hbai_household_net_income. Its declared engine reads are now what it materializes (UK_WAS_LISA_ENGINE_PREDICTORS): num_adults and num_children, from the FRS family roles since #1095, leave the list. - sources.yaml carries the declarations and both stages' notes. source_stages.json is re-derived from it, the release input coverage manifest is regenerated (its source_manifest_sha256 fields), and so is the H2 fixture: sources/fixture.json carries the two stages' declarations and notes, and stage 52's notes from 019f5d401, which waited for the tool to run; uk_spine.json carries the two stages' new contract hashes. No synthetic fact in the fixture moves. - docs/uk-lisa-1003.md and a changelog fragment describe the concept. What moves: every WAS draw can move on both lines, and with them the Lifetime ISA draws and the later stages that read wealth (means-tested capital, CGT selection, property components, high-value support). The size is unmeasured; a sampled f010 arm measures it. was_wealth, was_lisa and every later spine node re-key. Tests: the WAS recipient reads the engine's HBAI income, not a household_net_income decoy, and the donor maps DVTotinc_bhcR8 to the same name; the packaged declarations name both sides; drift to the old engine read or a wrong direct map is refused; the LISA recipient and donor carry the WAS predictor and its declared engine reads are what it materializes. Co-Authored-By: Claude Opus 5.5 --- .../931-uk-was-hbai-net-income.changed.md | 4 + docs/uk-lisa-1003.md | 6 +- .../uk/release_input_coverage_manifest.json | 50 +++---- .../src/microcosm/build/uk/source_stages.json | 21 ++- .../src/microcosm/build/uk/spec/sources.yaml | 19 ++- .../microcosm/build/uk_runtime/was_lisa.py | 41 ++++-- .../microcosm/build/uk_runtime/was_wealth.py | 74 +++++++++-- .../tests/engine_free/uk/test_uk_was_lisa.py | 124 +++++++++++++++++- .../engine_free/uk/test_uk_was_wealth.py | 110 +++++++++++++++- .../parity/uk_spine/sources/fixture.json | 21 ++- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- 11 files changed, 375 insertions(+), 97 deletions(-) create mode 100644 changelog.d/931-uk-was-hbai-net-income.changed.md diff --git a/changelog.d/931-uk-was-hbai-net-income.changed.md b/changelog.d/931-uk-was-hbai-net-income.changed.md new file mode 100644 index 0000000000..1792cad95f --- /dev/null +++ b/changelog.d/931-uk-was-hbai-net-income.changed.md @@ -0,0 +1,4 @@ +The UK WAS wealth and Lifetime ISA stages (`was_wealth`, `was_lisa`) read household net income on HBAI's concept on both sides (#931), as the LCFS, ETB services and NTS stages do (microcosm#1164). The donor's income is WAS `DVTotinc_bhcR8`, net regular household income before housing costs less council tax. The recipient's is policyengine-uk's `hbai_household_net_income_before_reset_to_zero`, in place of `household_net_income`, which also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that `shareholding` spreads over the households the engine simulates, and adds benefits in kind. Both sides call the predictor `hbai_household_net_income`. + +- **What moves.** The WAS chain keeps its segments, order and seeds, but every WAS draw can move. So do the Lifetime ISA draws and the later stages that read wealth: means-tested capital, CGT selection, property components and high-value support. A sampled f010 arm measures the change. +- **Declarations.** `was_wealth` declares the donor column in its `direct_maps`, and the stage refuses a declared direct map its donor cleaning does not read. `was_lisa`'s declared engine reads now list what it materializes, without `num_adults` and `num_children`, which come from the FRS family roles. Its realised profile keys its income tertiles `by_hbai_household_net_income_tertile`, and the ownership coefficients `log1p_hbai_household_net_income`. diff --git a/docs/uk-lisa-1003.md b/docs/uk-lisa-1003.md index 14d03b9d03..0951a06121 100644 --- a/docs/uk-lisa-1003.md +++ b/docs/uk-lisa-1003.md @@ -106,7 +106,11 @@ and its own WAS financial draws before the LISA draw. weight each person carries. Predictors: five age groups (the public file has five-year age bands only), the logarithm of the person's earnings, household net income, gross financial wealth, cash ISA, stocks-and-shares ISA and - savings, private renting, sex and the number of children. The intercept is + savings, private renting, sex and the number of children. Household net + income is HBAI's concept on both sides, as in `was_wealth`: WAS + `DVTotinc_bhcR8` (net income before housing costs, less council tax) on the + donor, and policyengine-uk's HBAI net income before negative incomes are + reset to zero on the recipient (microcosm#931). The intercept is unpenalised, so the model's weighted mean probability on the donor equals the donor's weighted ownership share. A recipient aged 18 or over holds when a uniform keyed on the person id falls below their probability, so adding or diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 52b0e50179..dcc22d7743 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "843b17b3f14b32fcd3d7c3ad71e10d7a0b1321523319d1a18b4c3ad3429ec924", + "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 6d2f6b780a..6f47a13392 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3361,7 +3361,8 @@ "num_adults": "NumAdultR8", "num_children": "NumChildR8", "council_tax": "DVCTaxAmtAnnualR8", - "rental_income": "DVNetRentAmtAnnualR8_aggr" + "rental_income": "DVNetRentAmtAnnualR8_aggr", + "hbai_household_net_income": "DVTotinc_bhcR8" }, "derived": { "private_pension_wealth": "totalpenr8_aggr - dvvaldbt_scaper8_aggr", @@ -3385,7 +3386,7 @@ { "kind": "materialize_rules_engine_predictors", "predictors": [ - "household_net_income", + "hbai_household_net_income_before_reset_to_zero", "private_pension_income", "employment_income", "self_employment_income", @@ -3399,7 +3400,7 @@ "seed": 0, "n_estimators": 100, "predictors": [ - "household_net_income", + "hbai_household_net_income", "num_adults", "num_children", "private_pension_income", @@ -3590,7 +3591,7 @@ "other_residential_property_secured_debt", "non_residential_property_secured_debt" ], - "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095)." + "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, is one concept on both sides (microcosm#1164): the donor's DVTotinc_bhcR8, net regular household income before housing costs less council tax, and the recipient's policyengine-uk HBAI net income before the reset to zero (hbai_household_net_income_before_reset_to_zero, as the LCFS, ETB services and NTS stages read it); the engine's household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds benefits in kind (microcosm#931). The other WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095)." }, { "stage": "was_lisa", @@ -3709,7 +3710,7 @@ "employment_income" ], "household_predictors": [ - "household_net_income", + "hbai_household_net_income", "num_adults", "num_children", "gross_financial_wealth", @@ -3726,9 +3727,7 @@ { "kind": "materialize_rules_engine_predictors", "predictors": [ - "household_net_income", - "num_adults", - "num_children", + "hbai_household_net_income_before_reset_to_zero", "private_pension_income", "employment_income", "self_employment_income", @@ -3755,7 +3754,7 @@ ], "log1p_predictors": [ "employment_income", - "household_net_income", + "hbai_household_net_income", "gross_financial_wealth", "cash_isa", "stocks_and_shares_isa", @@ -3799,7 +3798,7 @@ "age_band", "is_female", "employment_income", - "household_net_income", + "hbai_household_net_income", "gross_financial_wealth", "savings", "cash_isa", @@ -3843,7 +3842,7 @@ "lifetime_isa_balance", "household_lifetime_isa_balance" ], - "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income, gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María's ruling of 29 October 2026)." + "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (on HBAI's concept, the was_wealth predictor: DVTotinc_bhcR8 on the donor, policyengine-uk's HBAI net income before the reset to zero on the recipient; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María's ruling of 29 October 2026)." }, { "stage": "regional_property_uprating", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 3bc572d399..f80bbd49af 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -2914,6 +2914,7 @@ stages: num_children: NumChildR8 council_tax: DVCTaxAmtAnnualR8 rental_income: DVNetRentAmtAnnualR8_aggr + hbai_household_net_income: DVTotinc_bhcR8 derived: private_pension_wealth: totalpenr8_aggr - dvvaldbt_scaper8_aggr corporate_wealth_excl_isa: DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr @@ -2933,7 +2934,7 @@ stages: region: GORR8 via incumbent REGIONS map - kind: materialize_rules_engine_predictors predictors: - - household_net_income + - hbai_household_net_income_before_reset_to_zero - private_pension_income - employment_income - self_employment_income @@ -2944,7 +2945,7 @@ stages: seed: 0 n_estimators: 100 predictors: - - household_net_income + - hbai_household_net_income - num_adults - num_children - private_pension_income @@ -3102,7 +3103,7 @@ stages: - unit_and_investment_trusts - other_residential_property_secured_debt - non_residential_property_secured_debt - notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor''s from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient''s from the frame''s tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household''s mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner''s or a renter''s buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor''s conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor''s range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine''s share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine''s council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household''s FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor''s net rent DVNetRentAmtAnnualR8_aggr against the household''s property_income (uk-data#486/#495, microcosm#1095).' + notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor''s from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient''s from the frame''s tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household''s mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner''s or a renter''s buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor''s conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor''s range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine''s share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, is one concept on both sides (microcosm#1164): the donor''s DVTotinc_bhcR8, net regular household income before housing costs less council tax, and the recipient''s policyengine-uk HBAI net income before the reset to zero (hbai_household_net_income_before_reset_to_zero, as the LCFS, ETB services and NTS stages read it); the engine''s household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds benefits in kind (microcosm#931). The other WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine''s council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household''s FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor''s net rent DVNetRentAmtAnnualR8_aggr against the household''s property_income (uk-data#486/#495, microcosm#1095).' - stage: was_lisa survey: Wealth and Assets Survey round 8 source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain). @@ -3183,7 +3184,7 @@ stages: sentinel_recode_columns: - employment_income household_predictors: - - household_net_income + - hbai_household_net_income - num_adults - num_children - gross_financial_wealth @@ -3196,9 +3197,7 @@ stages: basis: LISAs have been opened only by people aged 18 to 39 since 6 April 2017, so no holder was older than 44 by the end of round-8 fieldwork (March 2022); a holder recorded in an age band starting at 45 or above is a misreported ISA type and is recoded to a non-holder. About GBP 30,500 of contributions, government bonus and the one-off 2017-18 Help to Buy ISA transfer was attainable before returns, so a holder above GBP 40,000 stays a holder but leaves the balance fit. - kind: materialize_rules_engine_predictors predictors: - - household_net_income - - num_adults - - num_children + - hbai_household_net_income_before_reset_to_zero - private_pension_income - employment_income - self_employment_income @@ -3216,7 +3215,7 @@ stages: age_group_lower_bounds: [0, 25, 35, 45, 55] log1p_predictors: - employment_income - - household_net_income + - hbai_household_net_income - gross_financial_wealth - cash_isa - stocks_and_shares_isa @@ -3250,7 +3249,7 @@ stages: - age_band - is_female - employment_income - - household_net_income + - hbai_household_net_income - gross_financial_wealth - savings - cash_isa @@ -3280,7 +3279,7 @@ stages: nonnegative_outputs: - lifetime_isa_balance - household_lifetime_isa_balance - notes: 'Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor''s weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income, gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner''s keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC''s 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC''s payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household''s balances down pro rata, existing holders'' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María''s ruling of 29 October 2026).' + notes: 'Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor''s weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (on HBAI''s concept, the was_wealth predictor: DVTotinc_bhcR8 on the donor, policyengine-uk''s HBAI net income before the reset to zero on the recipient; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner''s keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC''s 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC''s payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household''s balances down pro rata, existing holders'' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María''s ruling of 29 October 2026).' - stage: regional_property_uprating survey: Public regional property reference source: MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5). diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py index aa9c7d63f9..1bbd76bf81 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py @@ -19,14 +19,15 @@ reordering rows never moves a draw: * Who holds: a weighted logistic regression with a light ridge penalty on age - group, the log of the person's earnings and of household net income, gross - financial wealth, cash ISA, stocks-and-shares ISA and savings, private - renting, sex and the number of children. The intercept is unpenalised, so the - model's weighted mean probability on the donor equals the donor's weighted - ownership share. A person holds when their uniform falls below their - probability. The house regime gate (a gradient-boosting classifier) is - miscalibrated for an outcome held by under one per cent of adults with about - 120 donor holders, which is why this part is a logistic. + group, the log of the person's earnings and of household net income (on + HBAI's concept, as ``was_wealth`` reads it), gross financial wealth, cash + ISA, stocks-and-shares ISA and savings, private renting, sex and the number + of children. The intercept is unpenalised, so the model's weighted mean + probability on the donor equals the donor's weighted ownership share. A + person holds when their uniform falls below their probability. The house + regime gate (a gradient-boosting classifier) is miscalibrated for an outcome + held by under one per cent of adults with about 120 donor holders, which is + why this part is a logistic. * How much: the regime-gated QRF fitted on the holders alone and drawn at the owner's keyed quantile (``predict_positive_from_uniforms``), so a balance is positive exactly when the person holds. @@ -75,6 +76,7 @@ lifetime_isa_paying_ages, load_hmrc_isa_statistics, ) +from microcosm.build.uk_runtime.household_income import UK_HBAI_NET_INCOME_PREDICTOR from microcosm.build.uk_runtime.national_frame import ( uk_household_mass_conservation_receipt, uk_household_weight_kind, @@ -87,6 +89,7 @@ support_clip_to_donor_with_receipt, ) from microcosm.build.uk_runtime.was_wealth import ( + UK_WAS_ENGINE_PREDICTORS, clean_was_household_table, ) from microcosm.build.uk_runtime.was_wealth import ( @@ -159,9 +162,11 @@ #: The household columns of the cleaned WAS household tab that condition the #: person imputation (``clean_was_household_table`` names; ``is_renting`` there -#: is the private-renter flag ``DVPriRntR8 == 1``). +#: is the private-renter flag ``DVPriRntR8 == 1``). The income is the household's +#: net income on HBAI's concept, as ``was_wealth`` reads it on both sides +#: (``DVTotinc_bhcR8``; the engine's HBAI net income before the reset to zero). HOUSEHOLD_PREDICTORS: tuple[str, ...] = ( - "household_net_income", + UK_HBAI_NET_INCOME_PREDICTOR, "num_adults", "num_children", "gross_financial_wealth", @@ -173,6 +178,12 @@ #: Engine inputs read at person grain on the recipients (the models use earnings #: only; self-employment income is not read, microcosm#1059 review round 1). PERSON_ENGINE_PREDICTORS: tuple[str, ...] = ("employment_income",) +#: Every engine variable the stage materializes, which its +#: ``materialize_rules_engine_predictors`` declares: the WAS household +#: predictors (``was_wealth.recipient_predictors``), then the person earnings. +UK_WAS_LISA_ENGINE_PREDICTORS: tuple[str, ...] = tuple( + dict.fromkeys((*UK_WAS_ENGINE_PREDICTORS, *PERSON_ENGINE_PREDICTORS)) +) PRIVATE_RENTER_TENURE = "RENT_PRIVATELY" #: Donor response classes (evidence only; never predictors). @@ -835,8 +846,8 @@ def recipient_predictors( household_predictors = was_household_predictors(frame, engine) by_household = pd.DataFrame( { - "household_net_income": household_predictors[ - "household_net_income" + UK_HBAI_NET_INCOME_PREDICTOR: household_predictors[ + UK_HBAI_NET_INCOME_PREDICTOR ].to_numpy(dtype=float), "num_adults": household_predictors["num_adults"].to_numpy(dtype=float), "num_children": household_predictors["num_children"].to_numpy(dtype=float), @@ -1402,7 +1413,9 @@ def _profile( labels = spec.group_labels() female = frame["is_female"].to_numpy(dtype=float) == 1.0 renter = frame[PRIVATE_RENTER_COLUMN].to_numpy(dtype=float) == 1.0 - tertile = _tertiles(frame["household_net_income"].to_numpy(dtype=float), weights) + tertile = _tertiles( + frame[UK_HBAI_NET_INCOME_PREDICTOR].to_numpy(dtype=float), weights + ) def share(mask: np.ndarray) -> float | None: return _weighted_share(holds[mask], weights[mask]) if mask.any() else None @@ -1417,7 +1430,7 @@ def share(mask: np.ndarray) -> float | None: "private_renter": share(renter), "other": share(~renter), }, - "by_household_net_income_tertile": { + f"by_{UK_HBAI_NET_INCOME_PREDICTOR}_tertile": { str(index + 1): share(tertile == index) for index in range(3) }, "adults": int(len(frame)), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py index ecc1f490b4..3ee7005f3d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py @@ -19,6 +19,10 @@ ) from microcosm.build.uk_runtime.frs_brma import _benunit_household_map from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_NET_INCOME_PREDICTOR, + UK_HBAI_NET_INCOME_VARIABLE, +) from microcosm.build.uk_runtime.national_frame import ( uk_household_mass_conservation_receipt, uk_household_weight_kind, @@ -67,7 +71,9 @@ f"tenure_{category}" for category in UK_TENURE_CATEGORIES[1:] ) UK_WAS_WEALTH_PREDICTORS = ( - "household_net_income", + # The household's net income on the survey's concept on both sides + # (``UK_WAS_ENGINE_PREDICTOR_NAMES``; microcosm#1164). + UK_HBAI_NET_INCOME_PREDICTOR, "num_adults", "num_children", "private_pension_income", @@ -87,12 +93,24 @@ #: (``uc_relationships.household_family_role_counts``), matching the donor's #: NumAdultR8 and NumChildR8. UK_WAS_ENGINE_PREDICTORS = ( - "household_net_income", + UK_HBAI_NET_INCOME_VARIABLE, "private_pension_income", "employment_income", "self_employment_income", "capital_income", ) +#: Engine variables read under another predictor name. The donor's income is +#: WAS ``DVTotinc_bhcR8``, net regular household income before housing costs +#: less council tax, so the recipient's is the engine's HBAI net income before +#: the reset to zero, under the name the LCFS, ETB services and NTS stages give +#: it (``household_income``; microcosm#1164). The engine's +#: ``household_net_income`` it replaces also takes off stamp duty, the TV +#: licence and the business rates and corporate stamp duty that +#: ``shareholding`` spreads over the households the engine simulates, and adds +#: benefits in kind, none of which the donor's income counts (microcosm#931). +UK_WAS_ENGINE_PREDICTOR_NAMES: Mapping[str, str] = MappingProxyType( + {UK_HBAI_NET_INCOME_VARIABLE: UK_HBAI_NET_INCOME_PREDICTOR} +) UK_WAS_DEBT_OUTPUT_COLUMNS = ("mortgage_debt", "consumer_debt") #: Columns drawn only inside a tenure stratum and zero by rule outside it #: (microcosm#1063): the main residence exists only for owner-occupiers, and @@ -255,7 +273,7 @@ } REGION_REMAP = {"NORTHERN_IRELAND": "WALES"} UK_WAS_ENGINE_PREDICTOR_ENTITIES: Mapping[str, str] = { - "household_net_income": "household", + UK_HBAI_NET_INCOME_VARIABLE: "household", "private_pension_income": "person", "employment_income": "person", "self_employment_income": "person", @@ -353,7 +371,10 @@ "DVHseDebtR8_sum": "other_houses_secured_debt", "DVBLtDebtR8_sum": "buy_to_let_secured_debt", "DVBldDebtR8_sum": "non_residential_property_secured_debt", - "DVTotinc_bhcR8": "household_net_income", + # Net regular household income before housing costs less council tax: + # HBAI's concept, under the recipient's predictor name + # (``UK_WAS_ENGINE_PREDICTOR_NAMES``). + "DVTotinc_bhcR8": UK_HBAI_NET_INCOME_PREDICTOR, "DVSaValR8_aggr": "savings", "vcarnr8": "num_vehicles", "Tot_LosR8_aggr": "total_loans", @@ -732,7 +753,9 @@ def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: Engine variables live at their native entity; person- and benunit-level predictors are summed to household grain, reproducing the incumbent's - ``map_to="household"`` semantics. + ``map_to="household"`` semantics. Each takes its predictor name from + ``UK_WAS_ENGINE_PREDICTOR_NAMES``, so the HBAI net income before the reset + to zero is the household income predictor. """ materialized = engine.materialize( @@ -743,26 +766,27 @@ def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: benunit = frame.table("benunit") household_ids = pd.Index(household["household_id"]) result = pd.DataFrame(index=household.index) - for predictor in UK_WAS_ENGINE_PREDICTORS: - entity = UK_WAS_ENGINE_PREDICTOR_ENTITIES[predictor] - declared = str(engine.variable_metadata(predictor).entity) + for variable in UK_WAS_ENGINE_PREDICTORS: + predictor = UK_WAS_ENGINE_PREDICTOR_NAMES.get(variable, variable) + entity = UK_WAS_ENGINE_PREDICTOR_ENTITIES[variable] + declared = str(engine.variable_metadata(variable).entity) if declared != entity: raise ValueError( - f"engine declares {predictor!r} at entity {declared!r}; " + f"engine declares {variable!r} at entity {declared!r}; " f"the WAS wealth stage expects {entity!r}." ) - values = np.asarray(materialized[predictor]) + values = np.asarray(materialized[variable]) if entity == "household": if values.shape != (len(household),): raise ValueError( - f"materialized {predictor!r} has shape {values.shape}; " + f"materialized {variable!r} has shape {values.shape}; " f"expected ({len(household)},)." ) result[predictor] = values elif entity == "benunit": if values.shape != (len(benunit),): raise ValueError( - f"materialized {predictor!r} has shape {values.shape}; " + f"materialized {variable!r} has shape {values.shape}; " f"expected ({len(benunit)},)." ) groups = benunit["benunit_id"].map(_benunit_household_map(person)) @@ -771,7 +795,7 @@ def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: else: if values.shape != (len(person),): raise ValueError( - f"materialized {predictor!r} has shape {values.shape}; " + f"materialized {variable!r} has shape {values.shape}; " f"expected ({len(person)},)." ) summed = ( @@ -1677,7 +1701,12 @@ def _donor_artifact(stage: SourceStageSpec) -> Mapping[str, Any]: def _assert_chain_declaration(stage: SourceStageSpec) -> None: - """The chain op must declare the strata and the derived totals the run uses.""" + """Refuse a declaration that drifts from what the run uses. + + The chain's strata, predictors, derived totals and appended draws, the + engine reads and the donor columns the cleaning step names must all be the + runtime's. + """ operation = next( op for op in stage.operations if op.kind == "fit_weighted_qrf_chain" @@ -1710,6 +1739,23 @@ def _assert_chain_declaration(stage: SourceStageSpec) -> None: "was_wealth engine predictors drifted: manifest declares " f"{declared_engine!r}, runtime materializes {UK_WAS_ENGINE_PREDICTORS!r}." ) + # Every donor column the cleaning step declares is one the cleaning reads + # under that name (matched in lower case, as the cleaning matches them), + # so the household income's donor column is declared beside its engine + # variable. + cleaned_from = {target: source.lower() for source, target in _RAW_TO_CLEAN.items()} + drifted_maps = { + str(target): str(source) + for op in stage.operations + if op.kind == "derive" + for target, source in dict(op.parameters.get("direct_maps", {})).items() + if cleaned_from.get(str(target)) != str(source).lower() + } + if drifted_maps: + raise ValueError( + "was_wealth direct_maps drifted: manifest maps " + f"{drifted_maps!r}, which the donor cleaning does not read." + ) declared_totals = { str(total): tuple(components) for total, components in dict( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py index e2f21136ef..a8e03fa9bd 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py @@ -4,6 +4,7 @@ import importlib.util import json +from collections.abc import Mapping from types import SimpleNamespace import numpy as np @@ -16,6 +17,10 @@ HMRCISAStatisticsError, load_hmrc_isa_statistics, ) +from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_NET_INCOME_PREDICTOR, + UK_HBAI_NET_INCOME_VARIABLE, +) from microcosm.build.uk_runtime.national_frame import ( uk_national_frame, uk_time_period, @@ -24,8 +29,11 @@ from microcosm.build.uk_runtime.was_lisa import ( BALANCE_COLUMN, HOUSEHOLD_BALANCE_COLUMN, + HOUSEHOLD_PREDICTORS, OWNERSHIP_COLUMN, + PERSON_ENGINE_PREDICTORS, RULE_IMPOSSIBLE_HOLDER, + UK_WAS_LISA_ENGINE_PREDICTORS, UK_WAS_LISA_MASS_CONSERVATION_REASON, UK_WAS_LISA_OUTPUT_COLUMNS, UK_WAS_LISA_STAGE_NAME, @@ -42,7 +50,13 @@ impute_lifetime_isa_ownership, recipient_predictors, ) -from microcosm.build.uk_runtime.was_wealth import UK_WAS_ENGINE_PREDICTOR_ENTITIES +from microcosm.build.uk_runtime.was_wealth import ( + UK_WAS_ENGINE_PREDICTOR_ENTITIES, + UK_WAS_ENGINE_PREDICTORS, +) +from microcosm.build.uk_runtime.was_wealth import ( + recipient_predictors as was_household_predictors, +) from test_support.microcosm_build.uk_isa_statistics import ( ISA_FACTS_VERIFY_OPERATION, LISA_ANCHOR_OPERATION, @@ -78,7 +92,7 @@ "sentinel_codes": [-9, -8, -7, -6], "sentinel_recode_columns": ["employment_income"], "household_predictors": [ - "household_net_income", + "hbai_household_net_income", "num_adults", "num_children", "gross_financial_wealth", @@ -103,7 +117,7 @@ "age_group_lower_bounds": [0, 25, 35, 45, 55], "log1p_predictors": [ "employment_income", - "household_net_income", + "hbai_household_net_income", "gross_financial_wealth", "cash_isa", "stocks_and_shares_isa", @@ -125,7 +139,7 @@ "age_band", "is_female", "employment_income", - "household_net_income", + "hbai_household_net_income", "gross_financial_wealth", "savings", "cash_isa", @@ -278,7 +292,9 @@ def _recipient_frame(households: int = 64, time_period: str = "2024"): "num_bedrooms": 1 + household % 4, "council_tax": 1_000.0 + 10.0 * household, "council_tax_rebate": 0.0, - "household_net_income": 20_000.0 + 1_000.0 * (household % 60), + UK_HBAI_NET_INCOME_VARIABLE: 20_000.0 + 1_000.0 * (household % 60), + # The engine's other net income, which the stage never reads. + "household_net_income": 1_000_000.0 + household, "is_renting": bool(household % 4 == 0), "tenure_type": tenures[household % 4], "gross_financial_wealth": wealth, @@ -391,9 +407,16 @@ def test_clean_refuses_an_undeclared_column() -> None: def test_declaration_drift_is_refused() -> None: + # The engine's household_net_income in place of the HBAI income. with pytest.raises(WASLISAError, match="household_predictors drifted"): WASLISAColumns.from_parameters( - {**CLEAN, "household_predictors": ["household_net_income"]} + { + **CLEAN, + "household_predictors": [ + "household_net_income", + *CLEAN["household_predictors"][1:], + ], + } ) with pytest.raises(WASLISAError, match="weighted_ridge_logistic"): OwnershipModelSpec.from_parameters( @@ -749,6 +772,95 @@ def test_committed_declaration_carries_the_tested_parameters() -> None: ) +def _declared_names(value: object): + """Every string a declaration holds, keys included.""" + + if isinstance(value, str): + yield value + elif isinstance(value, Mapping): + for key, item in value.items(): + yield str(key) + yield from _declared_names(item) + elif isinstance(value, list | tuple): + for item in value: + yield from _declared_names(item) + + +def test_household_income_is_the_was_wealth_survey_concept() -> None: + """The stage reads household net income as ``was_wealth`` does. + + The donor's is the WAS household tab's ``DVTotinc_bhcR8`` and the + recipient's the engine's HBAI net income before the reset to zero, both + under the name ``household_income`` gives that income; the engine's + ``household_net_income`` is never read (microcosm#931). The declared + engine reads are what the stage materializes, and the realised profile's + income tertiles carry the same name. + """ + + assert HOUSEHOLD_PREDICTORS[0] == UK_HBAI_NET_INCOME_PREDICTOR + assert UK_WAS_LISA_ENGINE_PREDICTORS == tuple( + dict.fromkeys((*UK_WAS_ENGINE_PREDICTORS, *PERSON_ENGINE_PREDICTORS)) + ) + assert UK_HBAI_NET_INCOME_VARIABLE in UK_WAS_LISA_ENGINE_PREDICTORS + committed = load_country_spec("uk").sources.stage_map()[UK_WAS_LISA_STAGE_NAME] + by_kind = { + operation.kind: dict(operation.parameters) for operation in committed.operations + } + engine_reads = by_kind["materialize_rules_engine_predictors"]["predictors"] + assert tuple(engine_reads) == UK_WAS_LISA_ENGINE_PREDICTORS + logged = by_kind["impute_lifetime_isa_ownership"]["log1p_predictors"] + assert UK_HBAI_NET_INCOME_PREDICTOR in logged + assert ( + UK_HBAI_NET_INCOME_PREDICTOR + in by_kind["impute_lifetime_isa_balance"]["predictors"] + ) + declared = { + name + for operation in committed.operations + for name in _declared_names(operation.parameters) + } + assert "household_net_income" not in declared + + # The recipient's income is the WAS stage's predictor, the engine's HBAI + # income; the frame's household_net_income decoy is never read. + frame = _recipient_frame() + recipient = recipient_predictors( + frame, _FakeEngine(), age_band_edges=CLEAN["age_band_edges"] + ) + household = frame.table("household") + was = pd.Series( + was_household_predictors(frame, _FakeEngine())[ + UK_HBAI_NET_INCOME_PREDICTOR + ].to_numpy(dtype=float), + index=household["household_id"].to_numpy(), + ) + np.testing.assert_array_equal( + recipient[UK_HBAI_NET_INCOME_PREDICTOR].to_numpy(dtype=float), + was.reindex(recipient["person_household_id"].to_numpy()).to_numpy(), + ) + np.testing.assert_array_equal( + was.to_numpy(), + household[UK_HBAI_NET_INCOME_VARIABLE].to_numpy(dtype=float), + ) + assert "household_net_income" not in recipient + + # The donor's is DVTotinc_bhcR8, through the WAS household cleaning. + donor = _donor().person + raw = _was_donor().set_index("CASER8")["DVTotinc_bhcR8"] + np.testing.assert_array_equal( + donor[UK_HBAI_NET_INCOME_PREDICTOR].to_numpy(dtype=float), + raw.reindex(donor["household_key"].to_numpy()).to_numpy(dtype=float), + ) + + # The realised profile's income tertiles carry the predictor's name. + transform = _transform() + transform(frame) + realised = transform.checkpoint_metadata()["evidence"]["realised"] + for side in ("recipient", "donor"): + assert f"by_{UK_HBAI_NET_INCOME_PREDICTOR}_tertile" in realised[side] + assert "by_household_net_income_tertile" not in realised[side] + + def _bounds_tool(): path = _PATHS.repository / "tools/build_uk_was_lisa_support_bounds.py" spec = importlib.util.spec_from_file_location( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py index b436e2e945..dcc2419d4a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py @@ -6,7 +6,12 @@ import pandas as pd import pytest +from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_NET_INCOME_PREDICTOR, + UK_HBAI_NET_INCOME_VARIABLE, +) from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.tenure_constants import UK_TENURE_CATEGORIES from microcosm.build.uk_runtime.was_wealth import ( @@ -17,6 +22,7 @@ UK_WAS_DIRECT_SHARE_FRACTION_COLUMN, UK_WAS_DRAWN_ONLY_COLUMNS, UK_WAS_ENGINE_PREDICTOR_ENTITIES, + UK_WAS_ENGINE_PREDICTOR_NAMES, UK_WAS_ENGINE_PREDICTORS, UK_WAS_HOLDER_STRATA, UK_WAS_NET_FINANCIAL_LIABILITIES, @@ -228,7 +234,9 @@ def _frame() -> object: "num_bedrooms": [3, 4], "council_tax": [1000.0, 1200.0], "council_tax_rebate": [250.0, 0.0], - "household_net_income": [50000.0, 60000.0], + UK_HBAI_NET_INCOME_VARIABLE: [50000.0, 60000.0], + # The engine's other net income, which the stage must never read. + "household_net_income": [1_000_000.0, 2_000_000.0], "tenure_type": ["OWNED_WITH_MORTGAGE", "OWNED_OUTRIGHT"], } ) @@ -433,7 +441,7 @@ def test_recipient_predictors_sum_person_and_benunit_variables_to_household() -> assert predictors["num_adults"].tolist() == [2.0, 1.0] assert predictors["num_children"].tolist() == [0.0, 1.0] assert predictors["rental_income"].tolist() == [100.0, 50.0] - assert predictors["household_net_income"].tolist() == [50000.0, 60000.0] + assert predictors[UK_HBAI_NET_INCOME_PREDICTOR].tolist() == [50000.0, 60000.0] # The donor records the bill paid, so the reduction comes off the # liability before council tax reduction (microcosm#1095). assert predictors["council_tax"].tolist() == [750.0, 1200.0] @@ -460,6 +468,57 @@ def variable_metadata(self, name): recipient_predictors(_frame(), _WrongEntityEngine()) +def test_household_income_is_the_survey_concept_on_both_sides() -> None: + """Donor and recipient read household net income on HBAI's concept. + + The donor's ``DVTotinc_bhcR8`` (net income before housing costs, less + council tax) and the recipient's HBAI net income before the reset to zero + both carry the predictor name ``household_income`` gives that income, the + one the LCFS, ETB services and NTS stages read (microcosm#1164). The + engine's ``household_net_income``, which also takes off taxes and adds + benefits in kind the survey's income leaves out, is never read + (microcosm#931). + """ + + import microcosm.build.uk_runtime.was_wealth as module + + assert UK_WAS_WEALTH_PREDICTORS[0] == UK_HBAI_NET_INCOME_PREDICTOR + assert UK_WAS_ENGINE_PREDICTORS[0] == UK_HBAI_NET_INCOME_VARIABLE + assert dict(UK_WAS_ENGINE_PREDICTOR_NAMES) == { + UK_HBAI_NET_INCOME_VARIABLE: UK_HBAI_NET_INCOME_PREDICTOR + } + assert UK_WAS_ENGINE_PREDICTOR_ENTITIES[UK_HBAI_NET_INCOME_VARIABLE] == "household" + for names in ( + UK_WAS_WEALTH_PREDICTORS, + UK_WAS_ENGINE_PREDICTORS, + UK_WAS_ENGINE_PREDICTOR_ENTITIES, + ): + assert "household_net_income" not in names + + # The recipient's predictor is the engine's HBAI income; the frame's + # household_net_income decoy holds other values and is never read. + predictors = recipient_predictors(_frame(), _FakeEngine()) + assert predictors[UK_HBAI_NET_INCOME_PREDICTOR].tolist() == [50000.0, 60000.0] + assert "household_net_income" not in predictors + + # The donor's DVTotinc_bhcR8 under the same name. + assert module._RAW_TO_CLEAN["DVTotinc_bhcR8"] == UK_HBAI_NET_INCOME_PREDICTOR + donor = clean_was_household_table(_raw_was()) + assert donor[UK_HBAI_NET_INCOME_PREDICTOR].tolist() == [50000.0, 60000.0] + assert "household_net_income" not in donor + + # The packaged declaration names both sides, and the stage accepts it. + stage = load_country_spec("uk").sources.stage_map()["was_wealth"] + declared = {operation.kind: operation.parameters for operation in stage.operations} + direct_maps = declared["derive"]["direct_maps"] + assert direct_maps[UK_HBAI_NET_INCOME_PREDICTOR] == "DVTotinc_bhcR8" + engine_reads = declared["materialize_rules_engine_predictors"]["predictors"] + assert tuple(engine_reads) == UK_WAS_ENGINE_PREDICTORS + chain_predictors = declared["fit_weighted_qrf_chain"]["predictors"] + assert tuple(chain_predictors) == UK_WAS_WEALTH_PREDICTORS + module._assert_chain_declaration(stage) + + def test_student_loan_waterfall_uses_household_ids_not_positions() -> None: person = _frame().table("person").copy() balances = pd.Series([900.0, 300.0], index=[1, 2]) @@ -894,6 +953,49 @@ def test_chain_declaration_drift_is_refused() -> None: with pytest.raises(ValueError, match="predictors drifted"): module._assert_chain_declaration(stage) + def stage_with(*operations: dict[str, object]) -> SourceStageSpec: + return SourceStageSpec.from_mapping( + { + "stage": "was_wealth", + "survey": "test", + "source": "test", + "grain": "household", + "artifacts": [], + "operations": list(operations), + "outputs": list(UK_WAS_WEALTH_OUTPUT_COLUMNS), + } + ) + + # The engine's household_net_income in place of the HBAI income. + materialize = _materialize_operation() + materialize["predictors"] = [ + "household_net_income", + *UK_WAS_ENGINE_PREDICTORS[1:], + ] + with pytest.raises(ValueError, match="engine predictors drifted"): + module._assert_chain_declaration(stage_with(materialize, _chain_operation())) + # A declared donor column the cleaning does not read under that name: the + # donor's income under the engine's household_net_income, and the right + # name read from another column. + for direct_maps in ( + {"household_net_income": "DVTotinc_bhcR8"}, + {UK_HBAI_NET_INCOME_PREDICTOR: "DVGIEMPR8_AGGR"}, + ): + derive = {"kind": "derive", "scope": "test", "direct_maps": direct_maps} + with pytest.raises(ValueError, match="direct_maps drifted"): + module._assert_chain_declaration( + stage_with(derive, _materialize_operation(), _chain_operation()) + ) + # Matched in lower case, as the cleaning matches the donor's columns. + derive = { + "kind": "derive", + "scope": "test", + "direct_maps": {UK_HBAI_NET_INCOME_PREDICTOR: "dvtotinc_bhcr8"}, + } + module._assert_chain_declaration( + stage_with(derive, _materialize_operation(), _chain_operation()) + ) + def test_private_pension_wealth_split_preserves_the_old_corporate_wealth_identity() -> ( None @@ -961,7 +1063,7 @@ def _synthetic_recipients(n: int = 80, seed: int = 1063) -> pd.DataFrame: category = np.resize(np.asarray(UK_TENURE_CATEGORIES), n) recipient = pd.DataFrame( { - "household_net_income": rng.uniform(5_000, 90_000, n), + UK_HBAI_NET_INCOME_PREDICTOR: rng.uniform(5_000, 90_000, n), "num_adults": rng.integers(1, 4, n), "num_children": rng.integers(0, 3, n), "private_pension_income": rng.uniform(0, 20_000, n), @@ -1215,6 +1317,6 @@ def draw(rows: pd.DataFrame, ids: np.ndarray) -> pd.DataFrame: draw, recipient, _ids(recipient), - perturbed_column="household_net_income", + perturbed_column=UK_HBAI_NET_INCOME_PREDICTOR, perturbed_value=12_345.0, ) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 3ff4a956b4..ab320fbd02 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -6568,7 +6568,7 @@ ], "grain": "person", "nonnegative_outputs": [], - "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine's attendance once, at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.", + "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine's attendance once, at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula's last step over every person in the frame, from the engine's own values at the calibration year, each decided by a household's own records, and its own parameters: household market income plus benefits less the members' income tax and National Insurance, each household's weight once per member, microdf's weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine before its geographic clones exist), the stage stores the engine's flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine's flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.", "operations": [ { "draw": "attends_private_school_random_draw", @@ -8223,7 +8223,7 @@ "lifetime_isa_balance", "household_lifetime_isa_balance" ], - "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income, gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (Mar\u00eda's ruling of 29 October 2026).", + "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (on HBAI's concept, the was_wealth predictor: DVTotinc_bhcR8 on the donor, policyengine-uk's HBAI net income before the reset to zero on the recipient; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (Mar\u00eda's ruling of 29 October 2026).", "operations": [ { "artifact_role": "isa_statistics_facts", @@ -8280,7 +8280,7 @@ "household_aggregate": "DVFLISAVR8_aggr", "household_key": "CASER8", "household_predictors": [ - "household_net_income", + "hbai_household_net_income", "num_adults", "num_children", "gross_financial_wealth", @@ -8319,9 +8319,7 @@ "consumed_only": true, "kind": "materialize_rules_engine_predictors", "predictors": [ - "household_net_income", - "num_adults", - "num_children", + "hbai_household_net_income_before_reset_to_zero", "private_pension_income", "employment_income", "self_employment_income", @@ -8346,7 +8344,7 @@ "kind": "impute_lifetime_isa_ownership", "log1p_predictors": [ "employment_income", - "household_net_income", + "hbai_household_net_income", "gross_financial_wealth", "cash_isa", "stocks_and_shares_isa", @@ -8388,7 +8386,7 @@ "age_band", "is_female", "employment_income", - "household_net_income", + "hbai_household_net_income", "gross_financial_wealth", "savings", "cash_isa", @@ -8473,7 +8471,7 @@ "other_residential_property_secured_debt", "non_residential_property_secured_debt" ], - "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095).", + "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, is one concept on both sides (microcosm#1164): the donor's DVTotinc_bhcR8, net regular household income before housing costs less council tax, and the recipient's policyengine-uk HBAI net income before the reset to zero (hbai_household_net_income_before_reset_to_zero, as the LCFS, ETB services and NTS stages read it); the engine's household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds benefits in kind (microcosm#931). The other WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095).", "operations": [ { "derived": { @@ -8497,6 +8495,7 @@ "direct_maps": { "council_tax": "DVCTaxAmtAnnualR8", "gross_financial_wealth": "HFINWR8_SUM", + "hbai_household_net_income": "DVTotinc_bhcR8", "main_residence_mortgage": "TotMortR8", "main_residence_value": "DVhvalueR8", "mortgage_debt": "HMortGR8", @@ -8524,7 +8523,7 @@ "consumed_only": true, "kind": "materialize_rules_engine_predictors", "predictors": [ - "household_net_income", + "hbai_household_net_income_before_reset_to_zero", "private_pension_income", "employment_income", "self_employment_income", @@ -8609,7 +8608,7 @@ "kind": "fit_weighted_qrf_chain", "n_estimators": 4, "predictors": [ - "household_net_income", + "hbai_household_net_income", "num_adults", "num_children", "private_pension_income", diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 02ac47c4ac..b8bc94fe0b 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_recipients","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_uc_lcw_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_uc_carer_element","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"98ca74949e07fcf0f858bc88f882f2a196c6bf6f768b0002d2d8d1ce40ce56cb","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_recipients","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_uc_lcw_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_uc_carer_element","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"2523bdaed96ad432bc3f6bca68ed5edcf8f0a44d66bf3f04a3afa9fa30e3987b","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","care_hours","carers_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","frs_care_recipients","frs_uc_carer_element","iidb_reported","incapacity_benefit_reported","is_claimant_or_partner","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"cares_for_severely_disabled_person","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"5a89f3cfa318efe309329371bcaf438a2a44d86428d5f2839370439629686cb9","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","cares_for_severely_disabled_person","esa_contrib_reported","esa_income_reported","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_full_time_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"17f97af529c76e807f709af26a4bd1dd11e103c0f885048a73c18e28b94b9b54","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","is_in_full_time_education","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"5bd9c080af0e151afc10462fed72d491bad50e2516f0067654cb06680f4329ae","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"cb0552c10c46b3045be3f92cffa062fc7d1d55353da648b155e0513264c7a2ff","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"c5f170931bf33d9e490c8b79c176994f0b6a01582ddecebde665745bc5d859a9","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"2c682605139c0a8f96e8ba45a2c03f506d3f758ef437841066a8fdb8b74269b6","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"fea5354cec3ac0f6289018fc79d0d1ebc971870205ecf9c710df7a7312db21c7","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","frs_uc_lcw_status","frs_uc_carer_element","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"cares_for_severely_disabled_person","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"88e459e60ca51cb5e53fd2335ad6db1736241125484024834c28b56feef28496","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"13dd8d3f30efbd1d67227442ffd4547b69225e0f89b0f6b7f305b786c25172a8","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"4c0bcaabf3695a75ec7e69d99b6b8c90570fd345067d77f1ea315f5508a88962","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"6f9a9dc796a43487e520910c6e50561e9394323c83934923321741018648c661","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"174f4b38228d98107bee1639d1a297a2748bb618e87c7bd0a6c869c6ab06f8fa","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"d1c8f12081507e28a2a4b532961b4996577ea972b78e30b39a61f41a7d35046c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"23cc497627946fde4875c27b5ed2cf94a9cb89d7bd64025c1f34f1ac3b9baaf6","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"cfe459095fbb1cdeb95596442291d0405d99de1bc544a2ee27b3b49f841be940","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6753039b8ce02c6e4bed6f6b00d52c8c2a38e9b9a5f19eb3821625767f4667c6","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_kwh","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_kwh","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"8b9c6bcea0ced75d344e059daf10b03f010764025326ec0ee713b3774d39c686","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"3a83840f41735987cf9c6d110c6afc31387680d74c03ccab578f03e08de9c36a","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"362a543dce4f0578b11083cef2f2790b1ed3a7a6e539b31c5fdf796e517e8f15","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_limited_capability.","id":"uc_limited_capability","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_limited_capability@1","mass":"conserve","outputs":[{"column":"uc_limited_capability_for_WRA","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_limited_capability_for_work","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_limited_capability","stage_contract_sha256":"a8be9ab834fc66961b26395cf93f57103af3d474fe84d778aca85514a632acf2","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"76f1ec765c8fa7b0bd9dbf01784e528e8a51bc19e0b75d7a9f09d047badc5dd9","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"d39bc7366bd2828323ee974b23beb22b732646e2f7b28a4039b5a5f067671ac6","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"c0a3eb7c406b55909f37485560c8b655eb9b95ed518471e14d4c3647c105239b","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"401df68fcfc15aa99024e480d197d07d5536d485181abcaad0a148c17b9b3f4c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"29198953c5522c9b996b981478909916870977a79e9f2feb2db7e9e72ef15de8","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"497c3d06f99f108f46d54b8e6144d363005585640da18ac53cee3093fb809279","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"ed54c55428e56d7597c57c8459b014aa43b2ff9495b903f0b789bd5e4264ca9a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"02be930e9900eb352fc29cda472d556ba0a52bd6ded6e81f166aa45f554c6f1a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"465ecb7410a0d0a421505ad48475e78e18115cba66333d300a9d60166f297d3c","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"3335763d91350fa7f938ead181af4659aaca44ac6566b31158ac1284a01d965c","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"d86f2d3466d641b5064a0ae68942a53ad3cca005eae0080cd0a937b9c4632348","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","electricity_kwh","gas_kwh","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","frs_care_recipients","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","frs_uc_lcw_status","frs_uc_carer_element","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","cares_for_severely_disabled_person","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","is_in_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","uc_limited_capability_for_WRA","uc_limited_capability_for_work","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"6bdc671fde73a35dcf32e78d492e695d97c1d2f83226108c77d774a91bf61785","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pip_points.","id":"pip_points","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","pip_dl_category","pip_m_category"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pip_points@1","mass":"conserve","outputs":[{"column":"pip_dl_points","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_points","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pip_points","stage_contract_sha256":"43f97d6ffe248c3038c31ed9767548c656c0668c6624e9da06b8b28f815f9cfd","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_spi_synthetic","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","pip_m_points","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"trading_allowance_claim_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_allowance_full_relief","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.177.7"],["policyengine-core","3.34.10"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"50582c6c9e3d5d6300ff8058e3fbc77ce6adf13a487b3c3cf7f674367fcf4d9d","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"2c682605139c0a8f96e8ba45a2c03f506d3f758ef437841066a8fdb8b74269b6","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"b67ec0b7c62b9c759c0615092cf111c3d671608648315d9657595df1e0ee80b1","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"174f4b38228d98107bee1639d1a297a2748bb618e87c7bd0a6c869c6ab06f8fa","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"be1a93225103ce878603d9b72e44cb7ef871d2443b95807de10caab4d36dce64","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"29198953c5522c9b996b981478909916870977a79e9f2feb2db7e9e72ef15de8","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"ed54c55428e56d7597c57c8459b014aa43b2ff9495b903f0b789bd5e4264ca9a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"d27f0a6bc984fbbe83d50d60a9037ee6a693062306039abfa2dc5418b971b817","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} From 9b3d8c714fc68baa650317e1f6d2970a86336442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:40:52 +0000 Subject: [PATCH 009/108] Describe the WAS income predictor as the engine's closest concept (#931) Review of 640b2cf21. Its text said the WAS and Lifetime ISA stages read one income concept on both sides. The code matches the ruling, but the two sides are not the same income: policyengine-uk's HBAI net income also counts HBAI's in-kind items (free school meals, school milk and fruit, the free TV licence's value, Healthy Start vouchers, the Tax-Free Childcare top-up) and the Warm Home Discount, and the WAS user guide defines the income behind DVTotinc_bhcR8 only as total net regular household income less the council tax paid, so how it treats pension contributions, maintenance paid, student loan repayments and negative incomes is not verified. - The was_wealth and was_lisa notes, docs/uk-lisa-1003.md, the changelog fragment and the code comments call the recipient's income the engine's closest concept to the donor's and name those residuals. The was_wealth notes keep the income among the predictors that are not fully like-for-like. - household_net_income is described as adding the imputed value of NHS, education and transport spending, not "benefits in kind", which HBAI's income also counts in part. - Only the LCFS and ETB services stages name the income as a predictor; NTS reads the same engine variable to build gross income. The comments and test docstrings say so, and the two tests are renamed to match. - The changelog's What moves adds the calibrated weights on both lines. - source_stages.json is re-derived from sources.yaml (the two notes), the release input coverage manifest is regenerated (its source_manifest_sha256 fields), and the H2 fixture's sources/fixture.json carries the two notes. uk_spine.json does not move: notes are outside the stage contract hash. What moves: nothing in the data or any contract hash. Text only. Co-Authored-By: Claude Opus 5.5 --- .../931-uk-was-hbai-net-income.changed.md | 4 +- docs/uk-lisa-1003.md | 13 +++-- .../uk/release_input_coverage_manifest.json | 50 +++++++++---------- .../src/microcosm/build/uk/source_stages.json | 4 +- .../src/microcosm/build/uk/spec/sources.yaml | 4 +- .../microcosm/build/uk_runtime/was_lisa.py | 14 +++--- .../microcosm/build/uk_runtime/was_wealth.py | 25 ++++++---- .../tests/engine_free/uk/test_uk_was_lisa.py | 7 +-- .../engine_free/uk/test_uk_was_wealth.py | 13 ++--- .../parity/uk_spine/sources/fixture.json | 4 +- 10 files changed, 77 insertions(+), 61 deletions(-) diff --git a/changelog.d/931-uk-was-hbai-net-income.changed.md b/changelog.d/931-uk-was-hbai-net-income.changed.md index 1792cad95f..e5538812dd 100644 --- a/changelog.d/931-uk-was-hbai-net-income.changed.md +++ b/changelog.d/931-uk-was-hbai-net-income.changed.md @@ -1,4 +1,4 @@ -The UK WAS wealth and Lifetime ISA stages (`was_wealth`, `was_lisa`) read household net income on HBAI's concept on both sides (#931), as the LCFS, ETB services and NTS stages do (microcosm#1164). The donor's income is WAS `DVTotinc_bhcR8`, net regular household income before housing costs less council tax. The recipient's is policyengine-uk's `hbai_household_net_income_before_reset_to_zero`, in place of `household_net_income`, which also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that `shareholding` spreads over the households the engine simulates, and adds benefits in kind. Both sides call the predictor `hbai_household_net_income`. +The UK WAS wealth and Lifetime ISA stages (`was_wealth`, `was_lisa`) match the WAS donor's household net income to the engine's closest concept, its HBAI net income (#931), which the LCFS and ETB services stages also read as a predictor (microcosm#1164). The donor's income is WAS `DVTotinc_bhcR8`, net regular household income before housing costs less council tax. The recipient's is policyengine-uk's `hbai_household_net_income_before_reset_to_zero`, HBAI income before housing costs and before negative incomes are reset to zero, in place of `household_net_income`, which also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that `shareholding` spreads over the households the engine simulates, and adds the imputed value of NHS, education and transport spending. Both sides call the predictor `hbai_household_net_income`. The two incomes are close but not the same: the recipient's also counts HBAI's in-kind items (free school meals, school milk and fruit, the free TV licence's value, Healthy Start vouchers and the Tax-Free Childcare top-up) and the Warm Home Discount, which WAS's regular income is not documented to include, and how WAS treats pension contributions, maintenance paid, student loan repayments and negative incomes is not verified. -- **What moves.** The WAS chain keeps its segments, order and seeds, but every WAS draw can move. So do the Lifetime ISA draws and the later stages that read wealth: means-tested capital, CGT selection, property components and high-value support. A sampled f010 arm measures the change. +- **What moves.** The WAS chain keeps its segments, order and seeds, but every WAS draw can move. So do the Lifetime ISA draws, the later stages that read wealth (means-tested capital, CGT selection, property components and high-value support), and so the calibrated weights on both lines. A sampled f010 arm measures the change. - **Declarations.** `was_wealth` declares the donor column in its `direct_maps`, and the stage refuses a declared direct map its donor cleaning does not read. `was_lisa`'s declared engine reads now list what it materializes, without `num_adults` and `num_children`, which come from the FRS family roles. Its realised profile keys its income tertiles `by_hbai_household_net_income_tertile`, and the ownership coefficients `log1p_hbai_household_net_income`. diff --git a/docs/uk-lisa-1003.md b/docs/uk-lisa-1003.md index 0951a06121..8e91a33103 100644 --- a/docs/uk-lisa-1003.md +++ b/docs/uk-lisa-1003.md @@ -107,10 +107,15 @@ and its own WAS financial draws before the LISA draw. five-year age bands only), the logarithm of the person's earnings, household net income, gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children. Household net - income is HBAI's concept on both sides, as in `was_wealth`: WAS - `DVTotinc_bhcR8` (net income before housing costs, less council tax) on the - donor, and policyengine-uk's HBAI net income before negative incomes are - reset to zero on the recipient (microcosm#931). The intercept is + income is the `was_wealth` predictor: WAS `DVTotinc_bhcR8` (net regular + income before housing costs, less council tax) on the donor, and on the + recipient the engine's closest concept, policyengine-uk's HBAI net income + before housing costs and before negative incomes are reset to zero + (microcosm#931). The two are close but not the same: the recipient's also + counts HBAI's in-kind items, such as free school meals and the free TV + licence's value, and the Warm Home Discount, and how WAS treats pension + contributions, maintenance paid and negative incomes is not verified. + The intercept is unpenalised, so the model's weighted mean probability on the donor equals the donor's weighted ownership share. A recipient aged 18 or over holds when a uniform keyed on the person id falls below their probability, so adding or diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index dcc22d7743..275e6ea789 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "90f05e29b096d21b499b92c5d279f77e899da6be04449472a0a61b69ca813690", + "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 6f47a13392..229b33d899 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3591,7 +3591,7 @@ "other_residential_property_secured_debt", "non_residential_property_secured_debt" ], - "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, is one concept on both sides (microcosm#1164): the donor's DVTotinc_bhcR8, net regular household income before housing costs less council tax, and the recipient's policyengine-uk HBAI net income before the reset to zero (hbai_household_net_income_before_reset_to_zero, as the LCFS, ETB services and NTS stages read it); the engine's household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds benefits in kind (microcosm#931). The other WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095)." + "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, matches the donor's DVTotinc_bhcR8, net regular household income before housing costs less council tax, to the engine's closest concept, policyengine-uk's HBAI net income before housing costs and before the reset to zero (hbai_household_net_income_before_reset_to_zero, which the LCFS, ETB and NTS stages also read; microcosm#1164). The engine's household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds the imputed value of NHS, education and transport spending (microcosm#931). The WAS and FRS predictor definitions, that income among them, are not fully like-for-like and are ported as-is: the recipient's income also counts HBAI's in-kind items (free school meals, school milk and fruit, the free TV licence's value, Healthy Start vouchers and the Tax-Free Childcare top-up) and the Warm Home Discount, which WAS's regular income is not documented to include, and how WAS treats pension contributions, maintenance paid, student loan repayments and negative incomes is not verified; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095)." }, { "stage": "was_lisa", @@ -3842,7 +3842,7 @@ "lifetime_isa_balance", "household_lifetime_isa_balance" ], - "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (on HBAI's concept, the was_wealth predictor: DVTotinc_bhcR8 on the donor, policyengine-uk's HBAI net income before the reset to zero on the recipient; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María's ruling of 29 October 2026)." + "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (the was_wealth predictor: DVTotinc_bhcR8 on the donor and, on the recipient, the engine's closest concept, policyengine-uk's HBAI net income before the reset to zero, not fully like-for-like; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María's ruling of 29 October 2026)." }, { "stage": "regional_property_uprating", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index f80bbd49af..26a3adc4de 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3103,7 +3103,7 @@ stages: - unit_and_investment_trusts - other_residential_property_secured_debt - non_residential_property_secured_debt - notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor''s from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient''s from the frame''s tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household''s mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner''s or a renter''s buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor''s conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor''s range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine''s share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, is one concept on both sides (microcosm#1164): the donor''s DVTotinc_bhcR8, net regular household income before housing costs less council tax, and the recipient''s policyengine-uk HBAI net income before the reset to zero (hbai_household_net_income_before_reset_to_zero, as the LCFS, ETB services and NTS stages read it); the engine''s household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds benefits in kind (microcosm#931). The other WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine''s council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household''s FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor''s net rent DVNetRentAmtAnnualR8_aggr against the household''s property_income (uk-data#486/#495, microcosm#1095).' + notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor''s from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient''s from the frame''s tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household''s mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner''s or a renter''s buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor''s conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor''s range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine''s share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, matches the donor''s DVTotinc_bhcR8, net regular household income before housing costs less council tax, to the engine''s closest concept, policyengine-uk''s HBAI net income before housing costs and before the reset to zero (hbai_household_net_income_before_reset_to_zero, which the LCFS, ETB and NTS stages also read; microcosm#1164). The engine''s household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds the imputed value of NHS, education and transport spending (microcosm#931). The WAS and FRS predictor definitions, that income among them, are not fully like-for-like and are ported as-is: the recipient''s income also counts HBAI''s in-kind items (free school meals, school milk and fruit, the free TV licence''s value, Healthy Start vouchers and the Tax-Free Childcare top-up) and the Warm Home Discount, which WAS''s regular income is not documented to include, and how WAS treats pension contributions, maintenance paid, student loan repayments and negative incomes is not verified; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine''s council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household''s FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor''s net rent DVNetRentAmtAnnualR8_aggr against the household''s property_income (uk-data#486/#495, microcosm#1095).' - stage: was_lisa survey: Wealth and Assets Survey round 8 source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain). @@ -3279,7 +3279,7 @@ stages: nonnegative_outputs: - lifetime_isa_balance - household_lifetime_isa_balance - notes: 'Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor''s weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (on HBAI''s concept, the was_wealth predictor: DVTotinc_bhcR8 on the donor, policyengine-uk''s HBAI net income before the reset to zero on the recipient; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner''s keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC''s 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC''s payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household''s balances down pro rata, existing holders'' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María''s ruling of 29 October 2026).' + notes: 'Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor''s weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (the was_wealth predictor: DVTotinc_bhcR8 on the donor and, on the recipient, the engine''s closest concept, policyengine-uk''s HBAI net income before the reset to zero, not fully like-for-like; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner''s keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC''s 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC''s payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household''s balances down pro rata, existing holders'' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María''s ruling of 29 October 2026).' - stage: regional_property_uprating survey: Public regional property reference source: MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5). diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py index 1bbd76bf81..ad33f2f859 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py @@ -19,10 +19,11 @@ reordering rows never moves a draw: * Who holds: a weighted logistic regression with a light ridge penalty on age - group, the log of the person's earnings and of household net income (on - HBAI's concept, as ``was_wealth`` reads it), gross financial wealth, cash - ISA, stocks-and-shares ISA and savings, private renting, sex and the number - of children. The intercept is unpenalised, so the model's weighted mean + group, the log of the person's earnings and of household net income (as + ``was_wealth`` reads it: WAS's on the donor, the engine's closest concept, + its HBAI net income, on the recipient), gross financial wealth, cash ISA, + stocks-and-shares ISA and savings, private renting, sex and the number of + children. The intercept is unpenalised, so the model's weighted mean probability on the donor equals the donor's weighted ownership share. A person holds when their uniform falls below their probability. The house regime gate (a gradient-boosting classifier) is miscalibrated for an outcome @@ -163,8 +164,9 @@ #: The household columns of the cleaned WAS household tab that condition the #: person imputation (``clean_was_household_table`` names; ``is_renting`` there #: is the private-renter flag ``DVPriRntR8 == 1``). The income is the household's -#: net income on HBAI's concept, as ``was_wealth`` reads it on both sides -#: (``DVTotinc_bhcR8``; the engine's HBAI net income before the reset to zero). +#: net income as ``was_wealth`` reads it: ``DVTotinc_bhcR8`` on the donor, and +#: on the recipient the engine's closest concept, its HBAI net income before the +#: reset to zero. HOUSEHOLD_PREDICTORS: tuple[str, ...] = ( UK_HBAI_NET_INCOME_PREDICTOR, "num_adults", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py index 3ee7005f3d..d3f757b094 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py @@ -71,8 +71,8 @@ f"tenure_{category}" for category in UK_TENURE_CATEGORIES[1:] ) UK_WAS_WEALTH_PREDICTORS = ( - # The household's net income on the survey's concept on both sides - # (``UK_WAS_ENGINE_PREDICTOR_NAMES``; microcosm#1164). + # The household's net income: the donor's WAS income against the engine's + # closest concept (``UK_WAS_ENGINE_PREDICTOR_NAMES``; microcosm#1164). UK_HBAI_NET_INCOME_PREDICTOR, "num_adults", "num_children", @@ -101,13 +101,20 @@ ) #: Engine variables read under another predictor name. The donor's income is #: WAS ``DVTotinc_bhcR8``, net regular household income before housing costs -#: less council tax, so the recipient's is the engine's HBAI net income before -#: the reset to zero, under the name the LCFS, ETB services and NTS stages give -#: it (``household_income``; microcosm#1164). The engine's +#: less council tax, so the recipient's is the engine's closest concept, its +#: HBAI net income before the reset to zero, under the name the LCFS and ETB +#: services stages give it; NTS reads the same engine variable to build gross +#: income (``household_income``; microcosm#1164). The engine's #: ``household_net_income`` it replaces also takes off stamp duty, the TV #: licence and the business rates and corporate stamp duty that #: ``shareholding`` spreads over the households the engine simulates, and adds -#: benefits in kind, none of which the donor's income counts (microcosm#931). +#: the imputed value of NHS, education and transport spending, none of which +#: the donor's income counts (microcosm#931). The two are still not the same: +#: the HBAI income also counts HBAI's in-kind items (free school meals, the +#: free TV licence's value and the like) and the Warm Home Discount, which +#: WAS's regular income is not documented to include, and how WAS treats +#: pension contributions, maintenance paid, student loan repayments and +#: negative incomes is not verified. UK_WAS_ENGINE_PREDICTOR_NAMES: Mapping[str, str] = MappingProxyType( {UK_HBAI_NET_INCOME_VARIABLE: UK_HBAI_NET_INCOME_PREDICTOR} ) @@ -371,9 +378,9 @@ "DVHseDebtR8_sum": "other_houses_secured_debt", "DVBLtDebtR8_sum": "buy_to_let_secured_debt", "DVBldDebtR8_sum": "non_residential_property_secured_debt", - # Net regular household income before housing costs less council tax: - # HBAI's concept, under the recipient's predictor name - # (``UK_WAS_ENGINE_PREDICTOR_NAMES``). + # Net regular household income before housing costs less council tax, + # under the name of the recipient's closest engine concept, its HBAI net + # income (``UK_WAS_ENGINE_PREDICTOR_NAMES``). "DVTotinc_bhcR8": UK_HBAI_NET_INCOME_PREDICTOR, "DVSaValR8_aggr": "savings", "vcarnr8": "num_vehicles", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py index a8e03fa9bd..c7cd0485f3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py @@ -786,12 +786,13 @@ def _declared_names(value: object): yield from _declared_names(item) -def test_household_income_is_the_was_wealth_survey_concept() -> None: +def test_household_income_is_the_was_wealth_predictor() -> None: """The stage reads household net income as ``was_wealth`` does. The donor's is the WAS household tab's ``DVTotinc_bhcR8`` and the - recipient's the engine's HBAI net income before the reset to zero, both - under the name ``household_income`` gives that income; the engine's + recipient's the engine's closest concept, its HBAI net income before the + reset to zero, both under the name ``household_income`` gives that income + (the LCFS and ETB services stages give it the same name); the engine's ``household_net_income`` is never read (microcosm#931). The declared engine reads are what the stage materializes, and the realised profile's income tertiles carry the same name. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py index dcc2419d4a..c368d9f54f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py @@ -468,16 +468,17 @@ def variable_metadata(self, name): recipient_predictors(_frame(), _WrongEntityEngine()) -def test_household_income_is_the_survey_concept_on_both_sides() -> None: - """Donor and recipient read household net income on HBAI's concept. +def test_household_income_is_the_engine_hbai_income_under_one_name() -> None: + """The recipient reads the engine's closest concept to the donor's income. The donor's ``DVTotinc_bhcR8`` (net income before housing costs, less council tax) and the recipient's HBAI net income before the reset to zero both carry the predictor name ``household_income`` gives that income, the - one the LCFS, ETB services and NTS stages read (microcosm#1164). The - engine's ``household_net_income``, which also takes off taxes and adds - benefits in kind the survey's income leaves out, is never read - (microcosm#931). + name the LCFS and ETB services stages give it; NTS reads the same engine + variable to build gross income (microcosm#1164). The engine's + ``household_net_income``, which also takes off taxes the survey's income + leaves out and adds the imputed value of NHS, education and transport + spending, is never read (microcosm#931). """ import microcosm.build.uk_runtime.was_wealth as module diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index ab320fbd02..9d079c4d6c 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -8223,7 +8223,7 @@ "lifetime_isa_balance", "household_lifetime_isa_balance" ], - "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (on HBAI's concept, the was_wealth predictor: DVTotinc_bhcR8 on the donor, policyengine-uk's HBAI net income before the reset to zero on the recipient; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (Mar\u00eda's ruling of 29 October 2026).", + "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (the was_wealth predictor: DVTotinc_bhcR8 on the donor and, on the recipient, the engine's closest concept, policyengine-uk's HBAI net income before the reset to zero, not fully like-for-like; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (Mar\u00eda's ruling of 29 October 2026).", "operations": [ { "artifact_role": "isa_statistics_facts", @@ -8471,7 +8471,7 @@ "other_residential_property_secured_debt", "non_residential_property_secured_debt" ], - "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, is one concept on both sides (microcosm#1164): the donor's DVTotinc_bhcR8, net regular household income before housing costs less council tax, and the recipient's policyengine-uk HBAI net income before the reset to zero (hbai_household_net_income_before_reset_to_zero, as the LCFS, ETB services and NTS stages read it); the engine's household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds benefits in kind (microcosm#931). The other WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095).", + "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, matches the donor's DVTotinc_bhcR8, net regular household income before housing costs less council tax, to the engine's closest concept, policyengine-uk's HBAI net income before housing costs and before the reset to zero (hbai_household_net_income_before_reset_to_zero, which the LCFS, ETB and NTS stages also read; microcosm#1164). The engine's household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds the imputed value of NHS, education and transport spending (microcosm#931). The WAS and FRS predictor definitions, that income among them, are not fully like-for-like and are ported as-is: the recipient's income also counts HBAI's in-kind items (free school meals, school milk and fruit, the free TV licence's value, Healthy Start vouchers and the Tax-Free Childcare top-up) and the Warm Home Discount, which WAS's regular income is not documented to include, and how WAS treats pension contributions, maintenance paid, student loan repayments and negative incomes is not verified; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095).", "operations": [ { "derived": { From cefe9dfee916f2c3161f9633c02e92cd73dc121a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:44:12 +0000 Subject: [PATCH 010/108] Hold the corporate and total-economy land rows as measured diagnostics (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María's ruling of 6 Nov on the move, in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there. Two of the three ONS land rows cannot usefully bind weights: - ons.land.corporate_land_value (E43B, £2,063,534m): policyengine-uk's corporate_land_value hands out its aggregate by each household's share of the weighted corporate-sector wealth, so a released file meets that aggregate at any weights. Bound to the column measured at the starting weights, the row pinned only the weighted mass of the WAS corporate and private pension wealth, which no other row targets, not land. - ons.land.land_value (E44R, £6,872,455m) counts the land of financial corporations, government and NPISH, which the engine's land_value does not hold. With the household row (E44N, £4,485,171m) and the corporate row it asked for £323,750m that no weights could supply. Both get entries in uk/calibration_measure_exclusions.json in the obr.council_tax_england shape (tracking microcosm#1521, approved 2026-11-06, expiring 2027-02-06) and are measured on every evaluation. ons.land.household_land_value stays bound. Its binding note in uk/uk_population_targets.json said households_and_npish; E44N is the households sector alone, as the Chronicle package labels it, and the note now says so. The contract is hand-written. tools/generate_uk_target_references.py reads no binding note, so target_references.json and the membership report do not move (their regeneration test needs the pinned Chronicle feed, which is not on this machine, and skips here). What moves: the solve drops two rows on both lines, so calibrated weights move slightly; a calibrate-only rerun on the move's spine attributes it. Released corporate land value still equals the engine's aggregate. Tests: the packaged register's count and census take the two rows, and a new test pins their fields, the published values the reasons quote against target_reference_membership.json, the household row staying bound and its corrected note. Co-Authored-By: Claude Opus 5.5 --- .../931-uk-land-rows-diagnostic.changed.md | 3 + .../uk/calibration_measure_exclusions.json | 18 ++++ .../build/uk/uk_population_targets.json | 2 +- .../uk/test_uk_measure_simulation.py | 86 ++++++++++++++++++- 4 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 changelog.d/931-uk-land-rows-diagnostic.changed.md diff --git a/changelog.d/931-uk-land-rows-diagnostic.changed.md b/changelog.d/931-uk-land-rows-diagnostic.changed.md new file mode 100644 index 0000000000..25a46eccef --- /dev/null +++ b/changelog.d/931-uk-land-rows-diagnostic.changed.md @@ -0,0 +1,3 @@ +The UK calibration holds two of the three ONS national balance sheet land rows out of the solve as measured diagnostics (#931; María's ruling of 6 November 2026, workstream 2 of microcosm#1521). `ons.land.corporate_land_value` (E43B, non-financial corporations' land) reads policyengine-uk's `corporate_land_value`, which hands out the engine's own aggregate by each household's share of the weighted corporate-sector wealth: a released file meets that aggregate at any weights, and the solve, fitting the column measured at the starting weights, pinned only the weighted mass of the WAS corporate and private pension wealth, not land. `ons.land.land_value` (E44R, the total economy) counts the land of financial corporations, government and NPISH, which the engine's `land_value` does not hold, so beside the household (E44N) and corporate rows it asked for £323,750m that no weights could supply. Both are measured on every evaluation, under reviewed exclusions to 2027-02-06. `ons.land.household_land_value` stays bound, and its binding note now names the households sector, the one E44N covers, not households and NPISH. + +- **What moves.** The solve drops two rows on both lines, so the calibrated weights move slightly; a calibrate-only rerun on the move's spine measures it. The released corporate land value still equals the engine's aggregate, and the released land value moves only through household land. diff --git a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json index 3f2139ce0d..1f8e670c9f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json @@ -1368,6 +1368,24 @@ "adjudication": "María's go of 2026-11-12 on B10 of the November event-data plan's PR3 (microcosm#1095): P4 stays a stated limitation; measure Social Security Scotland's ADP and Carer Support Payment caseloads for Scotland, averaged over July to December 2025, on every build and hold them out of the solve (María's signature of 2026-11-12).", "approved_on": "2026-11-12", "expires_on": "2027-02-12" + }, + { + "name": "ons.land.corporate_land_value", + "reason": "Diagnostic by design, not a weight question: policyengine-uk's corporate_land_value gives each household aggregate_corporate_land_value times its share of the weighted corporate_sector_wealth (corporate and private pension wealth), so a released file's weighted total equals that parameter whatever the weights (policyengine-uk's test_corporate_land_value.py), and the row is met or missed by the parameter alone. The solve fits the column the engine measured at the spine's starting weights, so, bound, the row only pins the weighted mass of the corporate and private pension wealth the was_wealth stage draws, which no other row targets, and says nothing about land. Its fact, ONS's preliminary end-2025 national balance sheet E43B, non-financial corporations' land (£2,063,534m), is also not the engine's parameter, which policyengine-uk labels the land of non-financial and financial corporations. Measured on every evaluation; workstream 2 of microcosm#1521 settles the parameter's source.", + "tracking": "microcosm#1521", + "approved_by": "juaristi22", + "adjudication": "María's ruling of 2026-11-06 on the #931 move (microcosm#931), in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there (record the corporate row as met by construction and keep it out of the weight question): hold the corporate and total-economy land rows as measured diagnostics, keep ons.land.household_land_value bound and correct its note (E44N is the households sector alone, not households and NPISH). A calibrate-only rerun on the move's spine attributes the change.", + "approved_on": "2026-11-06", + "expires_on": "2027-02-06" + }, + { + "name": "ons.land.land_value", + "reason": "Diagnostic by design, not a weight question: ONS's preliminary end-2025 total-economy land (E44R, £6,872,455m) counts the land of financial corporations, general government and NPISH, which policyengine-uk's land_value, household_land_value plus corporate_land_value, does not represent. Bound beside the household row (E44N, households alone, £4,485,171m) and the corporate row (E43B, £2,063,534m), the three could not all be met: E44R less E44N less E43B is £323,750m, 4.7 percent of the total, which the solve could close only by raising property-weighted and corporate-wealth-weighted mass. Beside the bound household row, the rest of this row is the corporate column, which pins corporate-wealth mass rather than land (ons.land.corporate_land_value). The household row stays bound. Measured on every evaluation; workstream 2 of microcosm#1521 decides whether the engine gets the missing sectors as aggregates.", + "tracking": "microcosm#1521", + "approved_by": "juaristi22", + "adjudication": "María's ruling of 2026-11-06 on the #931 move (microcosm#931), in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there (record the corporate row as met by construction and keep it out of the weight question): hold the corporate and total-economy land rows as measured diagnostics, keep ons.land.household_land_value bound and correct its note (E44N is the households sector alone, not households and NPISH). A calibrate-only rerun on the move's spine attributes the change.", + "approved_on": "2026-11-06", + "expires_on": "2027-02-06" } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json index 132571e893..3ea46a15ec 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json @@ -16834,7 +16834,7 @@ "policyengine": { "metric_name": "ons/household_land_value", "value_variable": "household_land_value", - "notes": "National balance sheet land series for the households_and_npish sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:54-59 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." + "notes": "National balance sheet land series for the households sector (ONS E44N, households alone: NPISH land, like financial corporations' and government's, counts only in the total-economy row, ons.land.land_value); uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:54-59 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/household_land_value", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py index 0dadb0b4d5..b14e5e1388 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py @@ -755,6 +755,11 @@ def test_exclusion_applier_warns_within_week_of_expiry(): "obr.spirits_duties": 1, "obr.tobacco_duties": 1, "obr.betting_and_gaming_duties": 1, + # The #931 move (2026-11-06, microcosm#1521 workstream 2): the corporate + # row is met by construction, so bound it pins corporate-wealth mass, not + # land, and the total-economy row counts sectors the engine does not hold. + # Both are measured diagnostics; the household row stays bound. + "ons.land.": 2, } # The carer and childcare rows were retired on 2026-09-16 with the repairs @@ -813,8 +818,10 @@ def test_packaged_exclusions_load(): # ONS's NEET count (Y3). PR3 holds PAYE RTI's payrolled employees aged 0 to # 17 and 18 to 24 (B1), DWP's people on UC aged 16 to 21 and 22 to 24 in two # conditionality groups (B5), and Social Security Scotland's ADP and Carer - # Support Payment caseloads (B10). - assert len(names) == len(set(names)) == 143 + # Support Payment caseloads (B10). The #931 move holds the corporate and + # total-economy ONS land rows as measured diagnostics (María's ruling of + # 2026-11-06, microcosm#1521 workstream 2). + assert len(names) == len(set(names)) == 145 (gas,) = [ entry for entry in exclusions @@ -1162,6 +1169,81 @@ def test_employer_nics_are_a_measured_diagnostic(): assert "obr.ni_self_employed" not in exclusions +def test_the_corporate_and_total_land_rows_are_measured_diagnostics(): + """The #931 move holds two ONS land rows out of the solve (María's ruling + of 2026-11-06, microcosm#1521 workstream 2). + + policyengine-uk's corporate_land_value hands out its aggregate by each + household's share of the weighted corporate-sector wealth, so a released + file meets it at any weights and, bound, the row pins only that wealth's + mass. The total-economy row counts sectors the engine's land_value does + not hold, so the three land rows cannot all be met. The household row stays + bound, and its note names the one sector its series covers. + """ + + from importlib import resources as importlib_resources + + def resource(name: str) -> dict: + return json.loads( + importlib_resources.files("microcosm.build.uk") + .joinpath(name) + .read_text(encoding="utf-8") + ) + + exclusions = {e["name"]: e for e in load_uk_calibration_measure_exclusions()} + corporate = exclusions["ons.land.corporate_land_value"] + total = exclusions["ons.land.land_value"] + for entry in (corporate, total): + assert entry["tracking"] == "microcosm#1521", entry["name"] + assert entry["approved_by"] == "juaristi22", entry["name"] + # Three months, as the OBR council tax nation rows' window. + assert (entry["approved_on"], entry["expires_on"]) == ( + "2026-11-06", + "2027-02-06", + ), entry["name"] + assert entry["reason"].startswith("Diagnostic by design"), entry["name"] + assert "Measured on every evaluation" in entry["reason"], entry["name"] + assert "2026-11-06" in entry["adjudication"], entry["name"] + assert "workstream 2 of microcosm#1521" in entry["adjudication"] + assert "ons.land.household_land_value" not in exclusions + assert "corporate_sector_wealth" in corporate["reason"] + assert "was_wealth" in corporate["reason"] + + # The reasons quote the registry's published ONS series and their gap. + candidates = resource("target_reference_membership.json")["targets"] + + def published_millions(name: str) -> int: + (candidate,) = candidates[name]["candidates"] + return round(candidate["resolved_value"] / 1e6) + + series = { + "E44R": published_millions("ons.land.land_value"), + "E44N": published_millions("ons.land.household_land_value"), + "E43B": published_millions("ons.land.corporate_land_value"), + } + assert series == {"E44R": 6_872_455, "E44N": 4_485_171, "E43B": 2_063_534} + gap = series["E44R"] - series["E44N"] - series["E43B"] + assert gap == 323_750 + assert f"£{gap:,}m" in total["reason"] + for code, millions in series.items(): + assert code in total["reason"], code + assert f"£{millions:,}m" in total["reason"], code + assert "E43B" in corporate["reason"] + assert f"£{series['E43B']:,}m" in corporate["reason"] + + # The household row binds E44N, the households sector alone, and its note + # says so (it said households_and_npish). + contract = { + target["target_id"]: target + for target in resource("uk_population_targets.json")["targets"] + } + household = contract["ons.land.household_land_value"] + assert household["ledger_selector"]["source_concept"] == "ons.e44n" + note = household["bindings"]["policyengine"]["notes"] + assert "households sector (ONS E44N, households alone" in note + assert "households_and_npish" not in note + + def test_measure_resolver_direct_and_scratch_receipts(monkeypatch, tmp_path: Path): projections = stub_scratch_engine_window(monkeypatch) created = [] From 016797ea3b7c96eb9ab9624d257c3e0da6027f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:15:19 +0000 Subject: [PATCH 011/108] Test that the two land diagnostics leave the solve but stay measured (#931) Follow-up to 410e82283. Its test pinned the register's two entries, their fields and the published values they quote, but not what the entries do to a run. The new engine-free test compiles the three committed ONS land references with stand-in values and applies the packaged register's land entries, as both target loaders do after reconciliation. The solve surface keeps ons.land.household_land_value alone, the exclusion receipt carries the two entries' review fields, and the compiled register, which every run keeps beside the solve surface as its band-edge register (a national run writes it as national_contract_registry.json), still holds all three. On a synthetic four-household frame the national problem encoder binds the household row alone, with the receipt and the compiled register's digest in its bindings, while the compiled register materializes and scores all three rows at their weighted values. What moves: nothing. Test only. Co-Authored-By: Claude Opus 5.5 --- .../uk/test_uk_measure_simulation.py | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py index b14e5e1388..b01a1bebd6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py @@ -1244,6 +1244,165 @@ def published_millions(name: str) -> int: assert "households_and_npish" not in note +def test_the_two_land_diagnostics_are_measured_but_not_fitted(): + """The packaged register's two land entries take their rows out of the + national solve and leave them measured (the #931 move, María's ruling of + 2026-11-06). + + Both target loaders (load_uk_national_target_inputs and + load_uk_full_target_inputs) apply the register to the reconciled register: + the solve surface loses the corporate and total-economy rows, the run's + exclusion receipt carries their review, and the compiled register kept + beside the solve surface (the band-edge register; a national run writes it + as national_contract_registry.json) still holds all three. So the national + problem binds the household row alone, while the compiled register + materializes and scores the other two on the same frame. Synthetic weights + and land values. + """ + + from datetime import date + + from microcosm.build.country_spec import load_country_spec + from microcosm.build.uk_runtime.graph_national import encode_uk_national_problem + from microcosm.build.uk_runtime.ledger_surface import ( + _validate_band_edge_registry, + ) + from microcosm.build.uk_runtime.ledger_targets import ( + materialize_uk_ledger_targets, + ) + from microcosm.build.uk_runtime.national_calibration import ( + CalibrationFrameAdapter, + ) + from microcosm.build.uk_runtime.national_frame import uk_national_frame + from microcosm.calibrate import score_targets + from microcosm.calibrate.artifacts import decode_problem + from microcosm.frame import WeightKind + + land = ( + "ons.land.corporate_land_value", + "ons.land.household_land_value", + "ons.land.land_value", + ) + references = { + reference.name: reference + for reference in load_country_spec("uk").target_references + } + # The three committed land references, compiled with stand-in values. + compiled = TargetRegistry( + [ + TargetSpec( + name=name, + entity=references[name].entity, + measure=references[name].measure, + value=1_000.0, + source="test", + family=references[name].family, + period=references[name].period, + metadata=dict(references[name].metadata), + ) + for name in land + ], + country="uk", + ) + entries = tuple( + entry + for entry in load_uk_calibration_measure_exclusions() + if entry["name"] in land + ) + evaluated_on = max(date.fromisoformat(entry["approved_on"]) for entry in entries) + approved, receipt = apply_uk_calibration_measure_exclusions( + compiled, entries, now=evaluated_on + ) + assert [spec.name for spec in approved.specs] == ["ons.land.household_land_value"] + assert receipt == { + entry["name"]: { + **{field: value for field, value in entry.items() if field != "name"}, + "evaluated_on": evaluated_on.isoformat(), + } + for entry in entries + } + assert sorted(receipt) == ["ons.land.corporate_land_value", "ons.land.land_value"] + # The compiled register is the solve surface plus the receipt, as both + # loaders require before the solve. + _validate_band_edge_registry( + register_registry=approved, + band_edge_registry=compiled, + exclusion_receipt=receipt, + ) + + ids = np.arange(4, dtype="int64") + weights = np.array([10.0, 20.0, 30.0, 40.0]) + household_land = np.array([100.0, 200.0, 0.0, 50.0]) + corporate_land = np.array([5.0, 0.0, 15.0, 30.0]) + frame = uk_national_frame( + person=pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + } + ), + benunit=pd.DataFrame({"benunit_id": ids}), + household=pd.DataFrame( + { + "household_id": ids, + "source_household_id": ids, + "region": "LONDON", + "household_weight": weights, + "household_land_value": household_land, + "corporate_land_value": corporate_land, + "land_value": household_land + corporate_land, + } + ), + time_period="2025", + weight_kind=WeightKind.DESIGN, + ) + + # Not fitted: the national problem holds the household row alone and + # binds the receipt and the compiled register beside it. + problem = decode_problem( + encode_uk_national_problem( + frame, + national_registry=approved, + band_edge_registry=compiled, + period=2025, + doctrine={ + "target_weight_rule": "family_equal", + "target_loss_cap": 10.0, + "max_weight_ratio": 10.0, + "l0_lambda": 0.0, + "mass_rule": "free", + "scale_rule": "default_target_loss_scales", + }, + measure_exclusions=receipt, + resolver_factory=None, + ) + ) + assert problem.problem.names == ("ons.land.household_land_value@2025",) + np.testing.assert_array_equal(problem.problem.matrix.toarray(), [household_land]) + assert problem.bindings["measure_exclusions"] == receipt + assert problem.bindings["register_sha256"] == approved.version + assert problem.bindings["band_edge_register_sha256"] == compiled.version + + # Measured: the compiled register materializes on the same frame, and + # scoring it gives every land row its weighted estimate. + adapter = CalibrationFrameAdapter(frame) + materialized = materialize_uk_ledger_targets( + adapter, compiled, period=2025, band_edge_registry=compiled + ) + assert materialized.skipped == () + scored = score_targets(adapter.prepared_frame(), compiled.to_target_set()) + assert {row.name: row.final_estimate for row in scored.diagnostics} == ( + pytest.approx( + { + "ons.land.corporate_land_value@2025": weights @ corporate_land, + "ons.land.household_land_value@2025": weights @ household_land, + "ons.land.land_value@2025": weights @ (household_land + corporate_land), + } + ) + ) + + def test_measure_resolver_direct_and_scratch_receipts(monkeypatch, tmp_path: Path): projections = stub_scratch_engine_window(monkeypatch) created = [] From b5ce1ba8bd6bced01da803524a0834cebafe8b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:10:33 +0000 Subject: [PATCH 012/108] Follow up the review of the land diagnostics (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of 410e82283 and 161c77745. The two entries hold the right rows, but some of their text contradicted itself, the record of what moves was short, and one claim in 410e82283's message was wrong. - The total-economy reason, the changelog, the test docstring and the census comment now tie the £323,750m gap to the rows: E44R also counts the land of financial corporations, government and NPISH, which neither E44N nor E43B covers, and policyengine-uk's land_value is the sum of the two columns those rows read. The old text said the engine's land_value does not represent or hold financial corporations' land, while the corporate entry says policyengine-uk describes its corporate aggregate as the land of non-financial and financial corporations. That entry now says "describes as", not "labels". - The census comment no longer says the corporate row is met by construction: the column meets the engine's own aggregate at any weights, not E43B. The adjudications give Vahid's suggestion in his order (met by construction once the engine's parameter comes from the target's ONS series), and say the calibrate-only rerun is to attribute the change, since it has not run. - The two held rows' binding notes in uk/uk_population_targets.json record the hold, as the other held rows' notes do. The parity register's land concern names the two exclusions; uk_data_target_parity.json is regenerated with tools/census_uk_data_target_parity.py, and --check passes. The UK_BLOCK_SENSITIVE_MEASURE_COLUMNS comment and its test's comment no longer say the national targets bind the two variables. - 410e82283's message said the target-reference regeneration test skipped because the pinned Chronicle feed is not on this machine. The feed is here. tools/generate_uk_target_references.py, run with the test's arguments on that feed (facts and manifest hashes and the row count match uk/chronicle_feed.json) and on this commit's contract, reproduces target_references.json and target_reference_membership.json byte for byte. It ran directly, through the engine slot, rather than inside pytest, whose own load of the 10 GB feed would double the memory. The local generator keeps only local-level contract rows, so the land rows' notes cannot reach the local surface. What moves, all from 410e82283 (this commit is text, a regenerated parity register and tests): - Both lines split each family's equal share of the objective evenly over its bound rows (the dense line within each grain, where the land rows are amounts). ons.land.household_land_value, now the ons_land family's only bound row, carries the family's whole share: three times its loss weight before, on both lines. No other row's weight changes. The calibrate-only rerun measures this together with the two dropped rows. - The national frozen scoring register (national_target_registry.json, the approved register) loses the two rows, so a national run given a frozen register derived before 410e82283 refuses, as with obr.ni_employer. The dense loader checks the reconciled register, which keeps them. - The incumbent and release-cut scores, uk_target_fit and calibration_diagnostics.json read only the rows the solve binds, so the two rows leave them. - Neither line resolves land_value or corporate_land_value any more: the dense measure receipt's national_inputs falls by two, and a per-block dense run's block_sensitivity.present_in_this_run is empty. Evaluations that score the un-excluded register (tools/evaluate_uk_incumbent_surface.py) still measure the two rows, as they do every held row; 161c77745's test shows the compiled register can score them, doing the scoring itself. - The dense release lists the two exclusions among its limitations, each with the first 300 characters of its reason, which cite only published ONS figures and code facts. - The edited files are UK country-spec resources, so the national and dense target, measure and solve nodes re-key; full_measure.py feeds only those kernels. Spine nodes, source_stages.json, the release input coverage manifest and the H2 fixture do not move. The regional_home_values stage note still says the home value reaches three national rows, ons.land.land_value among them. It changes with group (c)'s rewrite of that stage, which regenerates source_stages.json and the H2 fixture anyway. Tests: test_the_corporate_and_total_land_rows_are_measured_diagnostics also pins the "4.7 percent" the total reason quotes and the two held notes. test_the_two_land_diagnostics_are_measured_but_not_fitted adds a bound row of another family (obr.tv_licence_fee) and pins the household row's family_equal weight at three times its share beside the other two land rows, with the other row's weight unchanged; its docstring says it applies the entries as the loaders do, without calling them. test_uk_measure_simulation, test_uk_full_measure, test_uk_data_target_parity, test_uk_population_targets, test_uk_incumbent_surface_evaluation, test_spec_source_citations, test_uk_target_references (its feed cases skip without CHRONICLE_UK_FACTS; the generator run above stands in for the national one) and test_uk_solve_scope pass, and ruff is clean. Co-Authored-By: Claude Opus 5.5 --- .../931-uk-land-rows-diagnostic.changed.md | 5 +- .../uk/calibration_measure_exclusions.json | 8 +- .../build/uk/uk_data_target_parity.json | 2 +- .../build/uk/uk_population_targets.json | 4 +- .../build/uk_runtime/data_target_parity.py | 2 +- .../build/uk_runtime/full_measure.py | 8 +- .../engine_free/uk/test_uk_full_measure.py | 5 +- .../uk/test_uk_measure_simulation.py | 95 ++++++++++++++----- 8 files changed, 91 insertions(+), 38 deletions(-) diff --git a/changelog.d/931-uk-land-rows-diagnostic.changed.md b/changelog.d/931-uk-land-rows-diagnostic.changed.md index 25a46eccef..a5c6e0a5da 100644 --- a/changelog.d/931-uk-land-rows-diagnostic.changed.md +++ b/changelog.d/931-uk-land-rows-diagnostic.changed.md @@ -1,3 +1,4 @@ -The UK calibration holds two of the three ONS national balance sheet land rows out of the solve as measured diagnostics (#931; María's ruling of 6 November 2026, workstream 2 of microcosm#1521). `ons.land.corporate_land_value` (E43B, non-financial corporations' land) reads policyengine-uk's `corporate_land_value`, which hands out the engine's own aggregate by each household's share of the weighted corporate-sector wealth: a released file meets that aggregate at any weights, and the solve, fitting the column measured at the starting weights, pinned only the weighted mass of the WAS corporate and private pension wealth, not land. `ons.land.land_value` (E44R, the total economy) counts the land of financial corporations, government and NPISH, which the engine's `land_value` does not hold, so beside the household (E44N) and corporate rows it asked for £323,750m that no weights could supply. Both are measured on every evaluation, under reviewed exclusions to 2027-02-06. `ons.land.household_land_value` stays bound, and its binding note now names the households sector, the one E44N covers, not households and NPISH. +The UK calibration holds two of the three ONS national balance sheet land rows out of the solve as measured diagnostics (#931; María's ruling of 6 November 2026, workstream 2 of microcosm#1521). `ons.land.corporate_land_value` (E43B, non-financial corporations' land) reads policyengine-uk's `corporate_land_value`, which hands out the engine's own aggregate by each household's share of the weighted corporate-sector wealth: a released file meets that aggregate at any weights, and the solve, fitting the column measured at the starting weights, pinned only the weighted mass of the WAS corporate and private pension wealth, not land. `ons.land.land_value` (E44R, the total economy) also counts the land of financial corporations, government and NPISH, which neither the household row (E44N) nor the corporate row covers, while the engine's `land_value` is the sum of the two columns those rows read, so beside them it asked for £323,750m that no weights could supply. Both are measured on every evaluation, under reviewed exclusions to 2027-02-06. `ons.land.household_land_value` stays bound, and its binding note now names the households sector, the one E44N covers, not households and NPISH; the two held rows' binding notes record the hold. -- **What moves.** The solve drops two rows on both lines, so the calibrated weights move slightly; a calibrate-only rerun on the move's spine measures it. The released corporate land value still equals the engine's aggregate, and the released land value moves only through household land. +- **What moves in the solve.** Two rows leave it on both lines. Both lines give each target family an equal share of the objective and split it evenly over the family's bound rows (on the dense line, within each grain, where the land rows are amounts), so `ons.land.household_land_value`, now the `ons_land` family's only bound row, carries the family's whole share, three times its loss weight before. No other row's weight changes, since the family keeps a row. The calibrated weights move slightly through both effects, which a calibrate-only rerun on the move's spine is to measure together. The released corporate land value still equals the engine's aggregate, and the released land value moves only through household land. +- **What else moves.** The national frozen scoring register (`national_target_registry.json`, the approved register) loses the two rows, so a national run given a frozen register derived before this change refuses, as with `obr.ni_employer`; the dense line checks its frozen register against the reconciled register, which keeps them. The incumbent and release-cut scores, `uk_target_fit` and `calibration_diagnostics.json` read only the rows the solve binds, so the two rows leave them. Neither line resolves `land_value` or `corporate_land_value` any more: the dense measure receipt's `national_inputs` falls by two, and a per-block dense run's `block_sensitivity.present_in_this_run` is empty. The dense release lists the two exclusions among its limitations (`measure_exclusion:`, with the first 300 characters of each reason). The edited files are UK country-spec resources, so the national and dense target, measure and solve nodes re-key; spine nodes, `source_stages.json` and the H2 spine fixture do not move. diff --git a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json index 1f8e670c9f..041a83f783 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json @@ -1371,19 +1371,19 @@ }, { "name": "ons.land.corporate_land_value", - "reason": "Diagnostic by design, not a weight question: policyengine-uk's corporate_land_value gives each household aggregate_corporate_land_value times its share of the weighted corporate_sector_wealth (corporate and private pension wealth), so a released file's weighted total equals that parameter whatever the weights (policyengine-uk's test_corporate_land_value.py), and the row is met or missed by the parameter alone. The solve fits the column the engine measured at the spine's starting weights, so, bound, the row only pins the weighted mass of the corporate and private pension wealth the was_wealth stage draws, which no other row targets, and says nothing about land. Its fact, ONS's preliminary end-2025 national balance sheet E43B, non-financial corporations' land (£2,063,534m), is also not the engine's parameter, which policyengine-uk labels the land of non-financial and financial corporations. Measured on every evaluation; workstream 2 of microcosm#1521 settles the parameter's source.", + "reason": "Diagnostic by design, not a weight question: policyengine-uk's corporate_land_value gives each household aggregate_corporate_land_value times its share of the weighted corporate_sector_wealth (corporate and private pension wealth), so a released file's weighted total equals that parameter whatever the weights (policyengine-uk's test_corporate_land_value.py), and the row is met or missed by the parameter alone. The solve fits the column the engine measured at the spine's starting weights, so, bound, the row only pins the weighted mass of the corporate and private pension wealth the was_wealth stage draws, which no other row targets, and says nothing about land. Its fact, ONS's preliminary end-2025 national balance sheet E43B, non-financial corporations' land (£2,063,534m), is also not the engine's parameter, which policyengine-uk describes as the land of non-financial and financial corporations. Measured on every evaluation; workstream 2 of microcosm#1521 settles the parameter's source.", "tracking": "microcosm#1521", "approved_by": "juaristi22", - "adjudication": "María's ruling of 2026-11-06 on the #931 move (microcosm#931), in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there (record the corporate row as met by construction and keep it out of the weight question): hold the corporate and total-economy land rows as measured diagnostics, keep ons.land.household_land_value bound and correct its note (E44N is the households sector alone, not households and NPISH). A calibrate-only rerun on the move's spine attributes the change.", + "adjudication": "María's ruling of 2026-11-06 on the #931 move (microcosm#931), in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there (record the corporate row as met by construction once the engine's parameter comes from the target's ONS series, and keep it out of the weight question): hold the corporate and total-economy land rows as measured diagnostics, keep ons.land.household_land_value bound and correct its note (E44N is the households sector alone, not households and NPISH). A calibrate-only rerun on the move's spine is to attribute the change.", "approved_on": "2026-11-06", "expires_on": "2027-02-06" }, { "name": "ons.land.land_value", - "reason": "Diagnostic by design, not a weight question: ONS's preliminary end-2025 total-economy land (E44R, £6,872,455m) counts the land of financial corporations, general government and NPISH, which policyengine-uk's land_value, household_land_value plus corporate_land_value, does not represent. Bound beside the household row (E44N, households alone, £4,485,171m) and the corporate row (E43B, £2,063,534m), the three could not all be met: E44R less E44N less E43B is £323,750m, 4.7 percent of the total, which the solve could close only by raising property-weighted and corporate-wealth-weighted mass. Beside the bound household row, the rest of this row is the corporate column, which pins corporate-wealth mass rather than land (ons.land.corporate_land_value). The household row stays bound. Measured on every evaluation; workstream 2 of microcosm#1521 decides whether the engine gets the missing sectors as aggregates.", + "reason": "Diagnostic by design, not a weight question: ONS's preliminary end-2025 total-economy land (E44R, £6,872,455m) also counts the land of financial corporations, general government and NPISH, which neither the household row (E44N, households alone, £4,485,171m) nor the corporate row (E43B, non-financial corporations, £2,063,534m) covers. policyengine-uk's land_value is household_land_value plus corporate_land_value, the columns those two rows read, so, bound beside them, the three rows could not all be met: E44R less E44N less E43B is £323,750m, 4.7 percent of the total, which the solve could close only by raising property-weighted and corporate-wealth-weighted mass. Beside the bound household row, the rest of this row is the corporate column, which pins corporate-wealth mass rather than land (ons.land.corporate_land_value). The household row stays bound. Measured on every evaluation; workstream 2 of microcosm#1521 decides whether the engine gets those sectors' land as aggregates.", "tracking": "microcosm#1521", "approved_by": "juaristi22", - "adjudication": "María's ruling of 2026-11-06 on the #931 move (microcosm#931), in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there (record the corporate row as met by construction and keep it out of the weight question): hold the corporate and total-economy land rows as measured diagnostics, keep ons.land.household_land_value bound and correct its note (E44N is the households sector alone, not households and NPISH). A calibrate-only rerun on the move's spine attributes the change.", + "adjudication": "María's ruling of 2026-11-06 on the #931 move (microcosm#931), in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there (record the corporate row as met by construction once the engine's parameter comes from the target's ONS series, and keep it out of the weight question): hold the corporate and total-economy land rows as measured diagnostics, keep ons.land.household_land_value bound and correct its note (E44N is the households sector alone, not households and NPISH). A calibrate-only rerun on the move's spine is to attribute the change.", "approved_on": "2026-11-06", "expires_on": "2027-02-06" } diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json b/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json index d124bd49af..2144b309b9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json @@ -1183,7 +1183,7 @@ "covers": [ "policyengine_uk_data.targets.sources.ons_land_values" ], - "evidence": "uk-data targets/sources/ons_land_values.py and _land.py; Microcosm target ids ons.land.{land_value,household_land_value,corporate_land_value}.", + "evidence": "uk-data targets/sources/ons_land_values.py and _land.py; Microcosm target ids ons.land.{land_value,household_land_value,corporate_land_value}, of which ons.land.land_value and ons.land.corporate_land_value are measure exclusions tracking microcosm#1521 since the #931 move.", "status": "ported_national" }, { diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json index 3ea46a15ec..9fa27c846b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json @@ -16800,7 +16800,7 @@ "policyengine": { "metric_name": "ons/land_value", "value_variable": "land_value", - "notes": "National balance sheet land series for the total_economy sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:48-52 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." + "notes": "National balance sheet land series for the total_economy sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:48-52 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Held as a measured diagnostic (calibration_measure_exclusions.json, María's ruling of 2026-11-06, microcosm#1521 workstream 2): E44R also counts the land of financial corporations, government and NPISH, which neither ons.land.household_land_value (E44N) nor ons.land.corporate_land_value (E43B) covers, and land_value is the sum of the two columns those rows read, so the three rows could not all be met." }, "axiom": { "metric_name": "ons/land_value", @@ -16868,7 +16868,7 @@ "policyengine": { "metric_name": "ons/corporate_land_value", "value_variable": "corporate_land_value", - "notes": "National balance sheet land series for the non_financial_corporations sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:60-65 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." + "notes": "National balance sheet land series for the non_financial_corporations sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:60-65 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Held as a measured diagnostic (calibration_measure_exclusions.json, María's ruling of 2026-11-06, microcosm#1521 workstream 2): policyengine-uk's corporate_land_value spreads the engine's own aggregate by each household's share of the weighted corporate-sector wealth, so the column meets that aggregate at any weights and, bound, the row pins corporate-wealth mass, not land." }, "axiom": { "metric_name": "ons/corporate_land_value", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py index f834985a8e..98505a762c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py @@ -340,7 +340,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "national_ons_land_values", "status": "ported_national", "classification": "red_line_national_family", - "evidence": "uk-data targets/sources/ons_land_values.py and _land.py; Microcosm target ids ons.land.{land_value,household_land_value,corporate_land_value}.", + "evidence": "uk-data targets/sources/ons_land_values.py and _land.py; Microcosm target ids ons.land.{land_value,household_land_value,corporate_land_value}, of which ons.land.land_value and ons.land.corporate_land_value are measure exclusions tracking microcosm#1521 since the #931 move.", }, { "concern_id": "regional_household_land_values", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py index 36a1403610..061c302f88 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py @@ -39,9 +39,11 @@ ) from microcosm.frame.adapters.policyengine_uk_population import UK_ENGINE_POPULATION -#: The engine variables the national targets bind that read policyengine-uk's -#: share formulas (``ons.land.corporate_land_value`` and ``ons.land.land_value``), -#: matched against the run's measure inputs. +#: The engine variables of the national rows that read policyengine-uk's share +#: formulas (``ons.land.corporate_land_value`` and ``ons.land.land_value``), +#: matched against the run's measure inputs. Both rows have been held out of +#: the solve since the #931 move (calibration_measure_exclusions.json), so a +#: run on the packaged register resolves neither variable. UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ("corporate_land_value", "land_value") #: What ``engine_population_representation.exact`` certifies for per-clone diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py index aa06f12c9b..0c07e1b109 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_measure.py @@ -308,8 +308,9 @@ def resolve(): } assert receipt["deviation"] == "per_clone_block_engine_resolution" sensitivity = receipt["block_sensitivity"] - # The engine variables the land rows bind, matched against the measure - # inputs (keyed by engine variable), not the rows' metric names. + # The engine variables the two share-formula land rows read, matched + # against the measure inputs (keyed by engine variable), not the rows' + # metric names. assert sensitivity["known_population_normalised_measures"] == [ "corporate_land_value", "land_value", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py index b01a1bebd6..4a377de01c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py @@ -756,9 +756,10 @@ def test_exclusion_applier_warns_within_week_of_expiry(): "obr.tobacco_duties": 1, "obr.betting_and_gaming_duties": 1, # The #931 move (2026-11-06, microcosm#1521 workstream 2): the corporate - # row is met by construction, so bound it pins corporate-wealth mass, not - # land, and the total-economy row counts sectors the engine does not hold. - # Both are measured diagnostics; the household row stays bound. + # column meets the engine's own aggregate at any weights, so bound it pins + # corporate-wealth mass, not land, and the total-economy row also counts + # sectors neither the household nor the corporate row covers. Both are + # measured diagnostics; the household row stays bound. "ons.land.": 2, } @@ -1176,9 +1177,11 @@ def test_the_corporate_and_total_land_rows_are_measured_diagnostics(): policyengine-uk's corporate_land_value hands out its aggregate by each household's share of the weighted corporate-sector wealth, so a released file meets it at any weights and, bound, the row pins only that wealth's - mass. The total-economy row counts sectors the engine's land_value does - not hold, so the three land rows cannot all be met. The household row stays - bound, and its note names the one sector its series covers. + mass. The total-economy row also counts sectors neither the household nor + the corporate row covers, and the engine's land_value is the sum of the two + columns those rows read, so the three land rows cannot all be met. The + household row stays bound, and its note names the one sector its series + covers; the two held rows' notes record the hold. """ from importlib import resources as importlib_resources @@ -1225,6 +1228,7 @@ def published_millions(name: str) -> int: gap = series["E44R"] - series["E44N"] - series["E43B"] assert gap == 323_750 assert f"£{gap:,}m" in total["reason"] + assert f"{100 * gap / series['E44R']:.1f} percent of the total" in total["reason"] for code, millions in series.items(): assert code in total["reason"], code assert f"£{millions:,}m" in total["reason"], code @@ -1242,6 +1246,14 @@ def published_millions(name: str) -> int: note = household["bindings"]["policyengine"]["notes"] assert "households sector (ONS E44N, households alone" in note assert "households_and_npish" not in note + assert "Held as a measured diagnostic" not in note + # The two held rows' notes record the hold, as the other held rows' do. + for name in ("ons.land.corporate_land_value", "ons.land.land_value"): + held = contract[name]["bindings"]["policyengine"]["notes"] + assert ( + "Held as a measured diagnostic (calibration_measure_exclusions.json, " + "María's ruling of 2026-11-06, microcosm#1521 workstream 2)" + ) in held, name def test_the_two_land_diagnostics_are_measured_but_not_fitted(): @@ -1249,15 +1261,18 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): national solve and leave them measured (the #931 move, María's ruling of 2026-11-06). - Both target loaders (load_uk_national_target_inputs and - load_uk_full_target_inputs) apply the register to the reconciled register: - the solve surface loses the corporate and total-economy rows, the run's - exclusion receipt carries their review, and the compiled register kept - beside the solve surface (the band-edge register; a national run writes it - as national_contract_registry.json) still holds all three. So the national - problem binds the household row alone, while the compiled register - materializes and scores the other two on the same frame. Synthetic weights - and land values. + The test applies the register's land entries to the compiled rows as both + target loaders (load_uk_national_target_inputs and + load_uk_full_target_inputs) apply the register to the reconciled register, + without calling either: the solve surface loses the corporate and + total-economy rows, the run's exclusion receipt carries their review, and + the compiled register kept beside the solve surface (the band-edge + register; a national run writes it as national_contract_registry.json) + still holds all three. So the national problem binds the household row as + the ons_land family's only row, with the family's whole share of the + objective, while the compiled register materializes and scores the other + two on the same frame. A bound row of another family (obr.tv_licence_fee) + stands beside them and keeps its share. Synthetic weights and values. """ from datetime import date @@ -1273,6 +1288,9 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): from microcosm.build.uk_runtime.national_calibration import ( CalibrationFrameAdapter, ) + from microcosm.build.uk_runtime.national_doctrine import ( + uk_national_target_loss_weights, + ) from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.calibrate import score_targets from microcosm.calibrate.artifacts import decode_problem @@ -1283,11 +1301,13 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): "ons.land.household_land_value", "ons.land.land_value", ) + other = "obr.tv_licence_fee" references = { reference.name: reference for reference in load_country_spec("uk").target_references } - # The three committed land references, compiled with stand-in values. + # The three committed land references and one bound row of another + # family, compiled with stand-in values. compiled = TargetRegistry( [ TargetSpec( @@ -1300,7 +1320,7 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): period=references[name].period, metadata=dict(references[name].metadata), ) - for name in land + for name in (*land, other) ], country="uk", ) @@ -1313,7 +1333,10 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): approved, receipt = apply_uk_calibration_measure_exclusions( compiled, entries, now=evaluated_on ) - assert [spec.name for spec in approved.specs] == ["ons.land.household_land_value"] + assert [spec.name for spec in approved.specs] == [ + "ons.land.household_land_value", + other, + ] assert receipt == { entry["name"]: { **{field: value for field, value in entry.items() if field != "name"}, @@ -1334,6 +1357,7 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): weights = np.array([10.0, 20.0, 30.0, 40.0]) household_land = np.array([100.0, 200.0, 0.0, 50.0]) corporate_land = np.array([5.0, 0.0, 15.0, 30.0]) + tv_licence = np.array([170.0, 170.0, 0.0, 170.0]) frame = uk_national_frame( person=pd.DataFrame( { @@ -1352,14 +1376,15 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): "household_land_value": household_land, "corporate_land_value": corporate_land, "land_value": household_land + corporate_land, + "tv_licence": tv_licence, } ), time_period="2025", weight_kind=WeightKind.DESIGN, ) - # Not fitted: the national problem holds the household row alone and - # binds the receipt and the compiled register beside it. + # Not fitted: the national problem holds the household row as the only + # land row and binds the receipt and the compiled register beside it. problem = decode_problem( encode_uk_national_problem( frame, @@ -1378,14 +1403,37 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): resolver_factory=None, ) ) - assert problem.problem.names == ("ons.land.household_land_value@2025",) - np.testing.assert_array_equal(problem.problem.matrix.toarray(), [household_land]) + assert problem.problem.names == ( + "ons.land.household_land_value@2025", + f"{other}@2025", + ) + np.testing.assert_array_equal( + problem.problem.matrix.toarray(), [household_land, tv_licence] + ) assert problem.bindings["measure_exclusions"] == receipt assert problem.bindings["register_sha256"] == approved.version assert problem.bindings["band_edge_register_sha256"] == compiled.version + # family_equal splits each family's equal share over its bound rows, so the + # household row, now the ons_land family's only one, carries the whole + # share: three times its weight beside the other two land rows. The other + # family's row keeps its weight. + beside = dict( + zip( + [spec.name for spec in compiled.specs], + uk_national_target_loss_weights( + [spec.family for spec in compiled.specs], rule="family_equal" + ), + strict=True, + ) + ) + assert problem.bindings["target_loss_weights"] == pytest.approx( + [3 * beside["ons.land.household_land_value"], beside[other]] + ) + assert problem.bindings["target_loss_weights"] == pytest.approx([0.5, 0.5]) # Measured: the compiled register materializes on the same frame, and - # scoring it gives every land row its weighted estimate. + # scoring it gives every row, the two held land rows included, its + # weighted estimate. adapter = CalibrationFrameAdapter(frame) materialized = materialize_uk_ledger_targets( adapter, compiled, period=2025, band_edge_registry=compiled @@ -1398,6 +1446,7 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): "ons.land.corporate_land_value@2025": weights @ corporate_land, "ons.land.household_land_value@2025": weights @ household_land, "ons.land.land_value@2025": weights @ (household_land + corporate_land), + f"{other}@2025": weights @ tv_licence, } ) ) From 22cde4b7b3221f46e0da47e4c319d0feb51cc163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:19:51 +0000 Subject: [PATCH 013/108] Mint UK copy ids from fixed radices that do not depend on K (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every UK copy layer took its id offset from ten to the digit count of the largest id it saw. Once (b) clones households at the spine's start, every offset would grow with K, and the K=1 national spine would stop being the clone-0 slice of the K=15 spine. Copy ids now come from fixed decimal fields declared in uk_runtime/copy_id_table.py, so every copy id is a function of its source id and its copy path, at today's stage order. The table is UK-local (María's rulings of 7 Nov); the shared table, a declaration digest and the int32 check come with the follow-up stage. - The table, least significant first: vintage (10 ordinals at 10**8, kept for pooled FRS vintages; nothing mints it), geographic_support (100 at 10**9), spi_support_channel (3 at 10**11: the SPI copy at 1, the band donor at 2), high_value_support (6 at 10**12), cgt_support_split (10,000 at 10**13), cgt_incidence_clone (2 at 10**17) and cgt_residential_split (8 at 10**18). The largest mintable id, about 7.2e18, is checked below int64 when the module loads. Helpers: radix, require_below, mint (refuses an ordinal outside its field, a row that already carries one and an int64 overflow), ordinal and parent. - Mint sites: frs_spine refuses raw ids at or above 10**8. Stages 16 (with its _remap_ids, which had no overflow check), 17 (sources read from the source-id columns), 26, 42, 43 and 46 mint through the table. Stage 42 refuses a household needing more than 10,000 copies with a fixed message. The anchor (45) pairs each twin by the twin field's radix. The executor takes each id-offset stage's declared radix, refuses an incumbent id at or above it and recovers parents on exact int64 ids; the row walk it replaces upcast all-numeric rows to float64, inexact above 2**53. Refusals print counts, never ids. The tests-only replace path mints through the table and refuses a base whose ids reach the SPI field. - Dense: the dense expand keeps its multiplier above the spine's largest id. A table-minted spine's twins and arms leave no int64 room above them, so every dense entry point (uk.full.expand, the ladder and crosswalk clone routes, the row-wise tool's plans) refuses n_clones > 1 in one message naming #931 and the clone-first spine. Dense K>1 builds wait for (c). K=1, and a spine without copy rows, expand as before; the expand kernel no longer multiplies an empty column by a multiplier above int64. - Declarations: the id_remapping texts in sources.yaml, source_stages.json and the code dicts name the table field. stack_zero_weight_donors and stack_income_band_donor_households gain the key, required by sources.schema.json and checked against the code. copy_id_table is bound into frs_spine and every minting stage, spi_support into stages 16 and 17, rowwise_geography into 43 and 45 and cgt_support into 26. What moves: base FRS rows keep their ids. Every copy row's five id columns change: SPI copies +10**11, band donors +2*10**11, high-value copies k*10**12, CGT copies k*10**13, twins +10**17, arms j*10**18. Relative id order is unchanged at K=1, so sorts, ties, ranks and folds are unchanged. Values move on copy rows through every draw keyed on the copy's own id: stage 18's stage-2 fill, stages 19-24, 27 and 28-41, every twin's Advani-Summers prior (so the CGT gains layer re-realises), stages 48-54 and policyengine-uk's warm_home_discount_random_draw on copy households. Base rows move second-order through pooled steps: stages 26-28's selections and walks, the residential solve, the anchor, the CGT walks and calibration. Draws keyed on person_source_id do not move. National ids pass 2**53 from stage 42 on. Dense K>1 builds are refused until (c). Every node re-keys. The coverage manifest's source digests and the H2 fixture are regenerated: the fixture's declarations, and its synthetic CGT asset-type, property purchase and rental facts, which are sized to the synthetic spine's copy-row values. Left for 1b: the identity tool's decoders still read the retired rule, so 11 tests in test_uk_identity_stability_receipts fail until 1b; also size_evaluation's decoders, the remaining id-printing messages and the stochastic_assignment binding sweep. Tests: the table's fields, bounds, order and ceiling, and the helpers' round trips and refusals; each copy stage's ids against its roots, found by lineage; a K=1 chain through all six layers sorts as the retired rule did and keeps base ids; clone-first K=1 and K=3 frames nest exactly and the retired rule does not (test_support/microcosm_build/ uk_clone_first_frames.py); the anchor pairs at a one-digit CGT ordinal; the executor's refusal and parents, exact above 2**53; the dense refusal and the K=1 path; the fixed CGT message; an AST guard against the retired rule; the binding tests. H2 parity passes on the regenerated fixture. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-copy-id-table.changed.md | 6 + .../spec_engine/schema/sources.schema.json | 8 + .../uk/release_input_coverage_manifest.json | 50 +- .../src/microcosm/build/uk/source_stages.json | 18 +- .../src/microcosm/build/uk/spec/sources.yaml | 16 +- .../build/uk_runtime/cgt_residential_split.py | 59 ++- .../build/uk_runtime/cgt_structure.py | 85 ++-- .../microcosm/build/uk_runtime/cgt_support.py | 56 ++- .../build/uk_runtime/copy_id_table.py | 249 ++++++++++ .../microcosm/build/uk_runtime/frs_spine.py | 24 + .../build/uk_runtime/graph_kernels.py | 170 ++++--- .../build/uk_runtime/graph_population.py | 7 +- .../uk_runtime/high_value_home_support.py | 50 +- .../build/uk_runtime/rowwise_dataset.py | 67 ++- .../build/uk_runtime/spi_band_donors.py | 34 +- .../microcosm/build/uk_runtime/spi_spine.py | 6 + .../microcosm/build/uk_runtime/spi_support.py | 74 +-- .../uk/test_uk_cgt_residential_split.py | 57 ++- .../engine_free/uk/test_uk_cgt_structure.py | 51 +- .../engine_free/uk/test_uk_cgt_support.py | 92 +++- .../engine_free/uk/test_uk_copy_id_table.py | 453 ++++++++++++++++++ .../tests/engine_free/uk/test_uk_graph.py | 73 +++ .../uk/test_uk_high_value_home_support.py | 37 ++ .../engine_free/uk/test_uk_spi_band_donors.py | 40 ++ .../engine_free/uk/test_uk_spi_support.py | 95 +++- .../sources/cgt_asset_type_facts.json | 56 +-- .../parity/uk_spine/sources/fixture.json | 16 +- .../sources/property_purchase_targets.json | 8 +- .../sources/property_rental_facts.json | 26 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- .../microcosm_build/uk_clone_first_frames.py | 333 +++++++++++++ tools/build_uk_rowwise_dataset.py | 21 + 32 files changed, 1966 insertions(+), 373 deletions(-) create mode 100644 changelog.d/931-uk-copy-id-table.changed.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py create mode 100644 test_support/microcosm_build/uk_clone_first_frames.py diff --git a/changelog.d/931-uk-copy-id-table.changed.md b/changelog.d/931-uk-copy-id-table.changed.md new file mode 100644 index 0000000000..cd35a29ce2 --- /dev/null +++ b/changelog.d/931-uk-copy-id-table.changed.md @@ -0,0 +1,6 @@ +The UK spine mints every copy row's ids from fixed decimal fields declared in `microcosm.build.uk_runtime.copy_id_table`, not from a power of ten above the largest id each copy layer sees (#931). Once the spine clones households at its start, that largest id would grow with the clone count K, so every layer's offset would move with K and the K=1 spine would stop being the clone-0 slice of the K=15 spine. Copy `k` in a field takes `id + k × radix` on all five id columns, so every copy id is a function of its source id and its copy path alone. The fields, least significant first: `vintage` (10 ordinals at 10^8, reserved for pooled FRS vintages, minted by nothing yet), `geographic_support` (100 at 10^9, for the clone-first spine), `spi_support_channel` (3 at 10^11: the SPI copy at 1, the income band donor at 2), `high_value_support` (6 at 10^12), `cgt_support_split` (10,000 at 10^13), `cgt_incidence_clone` (2 at 10^17) and `cgt_residential_split` (8 at 10^18, the top field). The largest mintable id is about 7.2e18, below int64's limit, which the module checks when it loads. + +- **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 rather than row by row, which upcast an all-numeric row to float64. Refusals print counts, never ids. +- **What moves.** Base FRS rows keep their ids. Every copy row's five id columns change: SPI copies by +10^11, band donors by +2×10^11, high-value copies by k×10^12, CGT copies by k×10^13, twins by +10^17 and arms by j×10^18. Relative id order is unchanged at K=1, so sorts, ties, ranks and folds are unchanged. Values move on copy rows through every draw keyed on the copy's own id: the SPI income stage's second stage, stages 19 to 24 and 27 to 41, every twin's Advani-Summers prior (so the CGT gains layer is re-realised), stages 48 to 54 and policyengine-uk's Warm Home Discount draw on copy households in spine engine calls. Base rows move second-order through pooled steps: the high-value selection, the regional value ranks, the landlord walk, the residential solve, the anchor, the CGT walks and calibration. Draws keyed on `person_source_id` do not move. National ids now pass 2^53 from the CGT support split on. +- **Dense line.** The dense expand keeps its multiplier above the spine's largest id, which a table-minted spine's twins and arms leave no room above: a dense pool with more than one clone is refused at every entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) in one message naming #931 and the clone-first spine. Dense builds with K > 1 wait for #931's clone-first spine; K = 1, and a spine without copy rows, expand as before. +- **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes, and the SPI support, band-donor, high-value and CGT clone and anchor stages also bind the helper modules they call. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index b70fa1faa7..91fc03a3ed 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -8271,6 +8271,7 @@ "count", "draw", "flag_column", + "id_remapping", "seed" ], "properties": { @@ -8283,6 +8284,9 @@ "draw": { "type": "string" }, + "id_remapping": { + "type": "string" + }, "flag_column": { "type": "string" }, @@ -10610,6 +10614,7 @@ "seed", "salt", "clone_index", + "id_remapping", "flag_column", "band_column", "carrier_column", @@ -10689,6 +10694,9 @@ "clone_index": { "type": "integer" }, + "id_remapping": { + "type": "string" + }, "flag_column": { "type": "string" }, diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 275e6ea789..1e48a779fa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ab87c6552f53173b935d977d69bd2189f917587a04e1678e7f550c42f6280ef7", + "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 229b33d899..daf1f8ff80 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -1684,7 +1684,8 @@ "base": "frs", "synthetic": "spi" }, - "draw": "uniform_without_replacement" + "draw": "uniform_without_replacement", + "id_remapping": "copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11)" }, { "kind": "gate_zero_weight_strata", @@ -1796,6 +1797,7 @@ "seed": 3, "salt": "spi_income_band_donor_draw", "clone_index": 2, + "id_remapping": "copy-id table field spi_support_channel; a donor takes every entity id + 2 x the field's radix (10**11), and its source is read from its source-id columns, never from the id", "flag_column": "household_is_spi_income_band_donor", "band_column": "spi_income_band_donor_lower_bound", "carrier_column": "person_is_spi_income_band_carrier", @@ -3902,7 +3904,7 @@ "copies": 6, "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", "copy_rule": "every selected household becomes copies rows at weight / copies", - "id_remapping": "id_multiplier_for_values; copy k takes every entity id + k x multiplier", + "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", "flag_column": "household_is_high_value_support_copy", "copies_column": "high_value_support_copies", "copy_index_column": "high_value_support_copy_index", @@ -7397,7 +7399,7 @@ "selection": "within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "maximum_copy_weight": 60.0, "copy_rule": "copies = ceil(household weight / maximum_copy_weight); every copy carries weight / copies", - "id_remapping": "id_multiplier_for_values; copy k takes every entity id + k x multiplier", + "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "flag_column": "household_is_cgt_support_copy", "copies_column": "cgt_support_copies", "copy_index_column": "cgt_support_copy_index", @@ -7441,7 +7443,7 @@ "mass_split": 0.5, "weight_kind_out": "importance", "conservation": "exact_total", - "id_remapping": "id_multiplier_for_values", + "id_remapping": "copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17)", "declared_factor": 1.0, "reason": "Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved." }, @@ -7719,8 +7721,8 @@ { "kind": "pair_clone_households_to_originals", "clone_flag_column": "household_is_capital_gains_clone", - "id_remapping": "id_multiplier_for_values", - "pairing": "clone household_id minus the clone stage's id multiplier, recomputed from the pre-clone rows (clone flag false; the support split's copies are pre-clone rows with their own clones)", + "id_remapping": "copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17)", + "pairing": "clone household_id minus the cgt_incidence_clone field's declared radix (the support split's copies are pre-clone rows with their own clones)", "requirement": "bijection between clone and original households; an unpaired clone or original fails the build" }, { @@ -7831,7 +7833,7 @@ "realization": "probability carried as weight, no draw: a household carrying k liable gainers becomes 2^k arms, one per subset of residential gainers, at the household's weight times the product of p over the residential gainers and 1 - p over the others; the arm with no residential gainer keeps the household's ids; at design weights the residential count and gains equal the solved expectations sum(p w) and sum(p w g) in every gain band", "maximum_liable_gainers_per_household": 3, "gainer_order": "person_id ascending within the household", - "id_remapping": "id_multiplier_for_values; arm j takes every entity id + j x multiplier, bit m of j marking the household's m-th liable gainer residential", + "id_remapping": "copy-id table field cgt_residential_split, the top field; arm j takes every entity id + j x the field's radix (10**18), bit m of j marking the household's m-th liable gainer residential", "flag_column": "household_is_cgt_residential_clone", "index_column": "cgt_residential_clone_index", "probability_column": "cgt_residential_probability", @@ -7855,7 +7857,7 @@ "cgt_residential_probability", "capital_gains_residential_property" ], - "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the id multiplier, the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time." + "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time." }, { "stage": "hmrc_cgt_asset_type_spine", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 26a3adc4de..2797dd92d8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -1523,6 +1523,7 @@ stages: base: frs synthetic: spi draw: uniform_without_replacement + id_remapping: copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11) - kind: gate_zero_weight_strata gate: uk_zero_weight_strata_gate declarations: @@ -1612,6 +1613,7 @@ stages: seed: 3 salt: spi_income_band_donor_draw clone_index: 2 + id_remapping: copy-id table field spi_support_channel; a donor takes every entity id + 2 x the field's radix (10**11), and its source is read from its source-id columns, never from the id flag_column: household_is_spi_income_band_donor band_column: spi_income_band_donor_lower_bound carrier_column: person_is_spi_income_band_carrier @@ -3324,7 +3326,7 @@ stages: copies: 6 stratum_rule: within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected copy_rule: every selected household becomes copies rows at weight / copies - id_remapping: id_multiplier_for_values; copy k takes every entity id + k x multiplier + id_remapping: copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12) flag_column: household_is_high_value_support_copy copies_column: high_value_support_copies copy_index_column: high_value_support_copy_index @@ -6010,7 +6012,7 @@ stages: selection: within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused maximum_copy_weight: 60.0 copy_rule: copies = ceil(household weight / maximum_copy_weight); every copy carries weight / copies - id_remapping: id_multiplier_for_values; copy k takes every entity id + k x multiplier + id_remapping: copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused flag_column: household_is_cgt_support_copy copies_column: cgt_support_copies copy_index_column: cgt_support_copy_index @@ -6045,7 +6047,7 @@ stages: mass_split: 0.5 weight_kind_out: importance conservation: exact_total - id_remapping: id_multiplier_for_values + id_remapping: copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17) declared_factor: 1.0 reason: Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved. - kind: draw_capital_gains_prior_from_banded_quantiles @@ -6270,8 +6272,8 @@ stages: operations: - kind: pair_clone_households_to_originals clone_flag_column: household_is_capital_gains_clone - id_remapping: id_multiplier_for_values - pairing: clone household_id minus the clone stage's id multiplier, recomputed from the pre-clone rows (clone flag false; the support split's copies are pre-clone rows with their own clones) + id_remapping: copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17) + pairing: clone household_id minus the cgt_incidence_clone field's declared radix (the support split's copies are pre-clone rows with their own clones) requirement: bijection between clone and original households; an unpaired clone or original fails the build - kind: derive_reporter_composition_from_banded_quantiles resource: advani_summers_capital_gains_distribution.json @@ -6362,7 +6364,7 @@ stages: realization: 'probability carried as weight, no draw: a household carrying k liable gainers becomes 2^k arms, one per subset of residential gainers, at the household''s weight times the product of p over the residential gainers and 1 - p over the others; the arm with no residential gainer keeps the household''s ids; at design weights the residential count and gains equal the solved expectations sum(p w) and sum(p w g) in every gain band' maximum_liable_gainers_per_household: 3 gainer_order: person_id ascending within the household - id_remapping: id_multiplier_for_values; arm j takes every entity id + j x multiplier, bit m of j marking the household's m-th liable gainer residential + id_remapping: copy-id table field cgt_residential_split, the top field; arm j takes every entity id + j x the field's radix (10**18), bit m of j marking the household's m-th liable gainer residential flag_column: household_is_cgt_residential_clone index_column: cgt_residential_clone_index probability_column: cgt_residential_probability @@ -6382,7 +6384,7 @@ stages: nonnegative_outputs: - cgt_residential_probability - capital_gains_residential_property - notes: 'Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row''s stake: at the top of the gains distribution one stake is several times a band''s expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household''s ids and arm j taking every entity id plus j times the id multiplier, the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone''s halves. Every household''s mass and composition and the total household mass are conserved; the receipt is gated at stage time.' + notes: 'Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row''s stake: at the top of the gains distribution one stake is several times a band''s expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household''s ids and arm j taking every entity id plus j times the radix of the copy-id table''s top field (microcosm#931), the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone''s halves. Every household''s mass and composition and the total household mass are conserved; the receipt is gated at stage time.' - stage: hmrc_cgt_asset_type_spine survey: HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed source: https://www.gov.uk/government/statistics/capital-gains-tax-statistics diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py index 2a4db3f39f..2a6e4f2c60 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py @@ -25,8 +25,9 @@ and ``_gains``; the by-band rows stay fenced), decides between the arms afterwards exactly as it decides between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; arm ``j`` takes every entity id plus -``j`` times the frame's id multiplier, the lineage rule the executor and the -geography identity kernel read for the support split and the clone. +``j`` times the declared radix of the copy-id table's top field, +``cgt_residential_split`` (microcosm#931), the lineage rule the executor +reads for the support split and the clone too. """ from __future__ import annotations @@ -39,6 +40,7 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_asset_type import ( CGT_RESIDENTIAL_GAINS_COLUMN, CGT_RESIDENTIAL_STOCK_SIGNAL, @@ -64,7 +66,6 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.build.uk_runtime.spi_support import ( _exact_total_correction, _importance_weights_with_exact_total, @@ -86,6 +87,9 @@ #: A household with more liable gainers than this would need more than eight #: arms; none exists on the FRS spine, so the stage refuses rather than grows. CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS = 3 +#: The copy-id field arm ``j`` takes ``j`` in: the table's top field, so every +#: arm id exceeds every incoming id (microcosm#931). +CGT_RESIDENTIAL_COPY_ID_FIELD = "cgt_residential_split" #: Concentration diagnostic: the share of the residential gains mass on the #: largest residential arm stakes (arm weight times gain). CGT_RESIDENTIAL_TOP_ARM_ROWS = 10 @@ -227,7 +231,8 @@ def split_cgt_residential_households( shift); a household with ``k`` liable gainers becomes ``2 ** k`` arms at product weights, the arm with no residential gainer keeping the household's ids and every other arm ``j`` taking every entity id plus - ``j`` times the frame's id multiplier. The exact-total correction lands on + ``j`` times the declared radix of the copy-id table's + ``cgt_residential_split`` field. The exact-total correction lands on the heaviest unsplit incumbent so every arm weight stays bitwise its product. Refuses a frame the stage has already split, one the asset-type stage has already classified, or a household with more liable gainers @@ -321,13 +326,17 @@ def split_cgt_residential_households( f"{CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS} liable gainers; the " "residential split refuses to enumerate their arms." ) - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + for table, columns in ( + (person, _PERSON_ID_COLUMNS), + (benunit, ("benunit_id",)), + (household, ("household_id",)), + ): + for column in columns: + copy_id_table.require_below( + table[column], + CGT_RESIDENTIAL_COPY_ID_FIELD, + what=f"incoming {column} values", + ) household_position = pd.Series(np.arange(len(household)), index=household_ids) incumbent_weights = weights.copy() arm_person: list[pd.DataFrame] = [] @@ -365,10 +374,11 @@ def split_cgt_residential_households( member_positions = np.flatnonzero(members) member_rank = rank_by_position.reindex(member_positions).to_numpy() for j in range(1, 2**k): - id_offset = j * multiplier copy_person = family_person.copy() for column in _PERSON_ID_COLUMNS: - copy_person[column] = copy_person[column].astype("int64") + id_offset + copy_person[column] = copy_id_table.mint( + copy_person[column], CGT_RESIDENTIAL_COPY_ID_FIELD, j + ) residential = np.zeros(len(copy_person), dtype=bool) for m in range(k): if (j >> m) & 1: @@ -377,12 +387,12 @@ def split_cgt_residential_households( residential, gains[member_positions], 0.0 ) copy_benunit = family_benunit.copy() - copy_benunit["benunit_id"] = ( - copy_benunit["benunit_id"].astype("int64") + id_offset + copy_benunit["benunit_id"] = copy_id_table.mint( + copy_benunit["benunit_id"], CGT_RESIDENTIAL_COPY_ID_FIELD, j ) copy_household = family_household.copy() - copy_household["household_id"] = ( - copy_household["household_id"].astype("int64") + id_offset + copy_household["household_id"] = copy_id_table.mint( + copy_household["household_id"], CGT_RESIDENTIAL_COPY_ID_FIELD, j ) copy_household[HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE] = True copy_household[CGT_RESIDENTIAL_CLONE_INDEX_COLUMN] = np.full( @@ -392,10 +402,10 @@ def split_cgt_residential_households( arm_benunit.append(copy_benunit) arm_household.append(copy_household) arm_weights.append(weights[positions] * factors[:, j]) - # Every arm id exceeds every incoming id (the multiplier is larger than - # any id), so the arms follow the incumbents; within the arms the rows are - # ordered by id so the group tables stay sorted whatever the household - # order the arms were built in. + # Every arm id exceeds every incoming id (the arms' field is the copy-id + # table's top field, above every incoming id), so the arms follow the + # incumbents; within the arms the rows are ordered by id so the group + # tables stay sorted whatever the household order the arms were built in. arms_person = pd.concat(arm_person, ignore_index=True) if arm_person else None arms_benunit = pd.concat(arm_benunit, ignore_index=True) if arm_benunit else None arms_household = ( @@ -716,9 +726,9 @@ def cgt_residential_split_operation_parameters() -> tuple[ ), "gainer_order": "person_id ascending within the household", "id_remapping": ( - "id_multiplier_for_values; arm j takes every entity id + j x " - "multiplier, bit m of j marking the household's m-th liable " - "gainer residential" + "copy-id table field cgt_residential_split, the top field; arm " + "j takes every entity id + j x the field's radix (10**18), bit " + "m of j marking the household's m-th liable gainer residential" ), "flag_column": HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, "index_column": CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, @@ -782,6 +792,7 @@ def _assert_cgt_residential_split_stage_parameters(stage: SourceStageSpec) -> No __all__ = [ "CGT_RESIDENTIAL_CLONE_INDEX_COLUMN", + "CGT_RESIDENTIAL_COPY_ID_FIELD", "CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS", "CGT_RESIDENTIAL_PROBABILITY_COLUMN", "CGT_RESIDENTIAL_SPLIT_MASS_CHANGE_REASON", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py index 2923099276..0498f1de67 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py @@ -19,6 +19,7 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.advani_summers import ( ADVANI_SUMMERS_RESOURCE, CGT_QUANTILE_POINTS, @@ -46,16 +47,20 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.rowwise_geography import ( - clone_entity_frame, - id_multiplier_for_values, -) +from microcosm.build.uk_runtime.rowwise_geography import clone_entity_frame from microcosm.build.uk_runtime.spi_support import ( _importance_weights_with_exact_total, ) from microcosm.frame import Frame, MassChangeRecord, WeightKind CGT_CLONE_MASS_SPLIT = 0.5 +#: The copy-id field the incidence twin takes ordinal 1 in (microcosm#931). +CGT_CLONE_COPY_ID_FIELD = "cgt_incidence_clone" +#: The id rule the clone and anchor manifests restate. +CGT_CLONE_ID_REMAPPING = ( + "copy-id table field cgt_incidence_clone; the twin takes every entity id + " + "the field's radix (10**17)" +) CGT_PRIOR_SEED = 0 CGT_PRIOR_SALT = "cgt_prior_amount" CGT_ADULT_MINIMUM_AGE = 16 @@ -249,13 +254,19 @@ def clone_cgt_incidence( person = frame.table("person").copy() benunit = frame.table("benunit").copy() household = frame.table("household").copy() - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + # The twin takes every id plus the declared radix of the copy-id table's + # cgt_incidence_clone field, which sits above every field filled before + # it (microcosm#931). + for table, columns in ( + (person, ("person_id", "person_household_id", "person_benunit_id")), + (benunit, ("benunit_id",)), + (household, ("household_id",)), + ): + for column in columns: + copy_id_table.require_below( + table[column], CGT_CLONE_COPY_ID_FIELD, what=f"incoming {column} values" + ) + multiplier = copy_id_table.radix(CGT_CLONE_COPY_ID_FIELD) cloned_person = clone_entity_frame( person, id_columns=("person_id", "person_household_id", "person_benunit_id"), @@ -343,12 +354,14 @@ def pair_clone_households( ) -> tuple[np.ndarray, np.ndarray, int]: """Positions of every clone household and of its paired original. - The clone stage offsets ids by ``id_multiplier_for_values`` over the rows - it saw; those rows are exactly the households whose clone flag is false, - the support split's copies included (they are pre-clone rows with their - own clones, microcosm#1045). Recomputing the multiplier from them - reproduces the executor's lineage rule, and the pairing must be a - bijection: an unpaired clone or original fails closed. + The clone stage offsets ids by the declared radix of the copy-id table's + ``cgt_incidence_clone`` field over the rows it saw; those rows are exactly + the households whose clone flag is false, the support split's copies + included (they are pre-clone rows with their own clones, microcosm#1045). + A clone's original is its id less that radix, the executor's lineage rule, + whatever the frame's largest id (microcosm#931), and the pairing must be + a bijection: an unpaired clone or original fails closed, printing counts + only. """ if HOUSEHOLD_IS_CGT_CLONE not in household.columns: @@ -365,31 +378,22 @@ def pair_clone_households( ) if len(np.unique(household_ids)) != len(household_ids): raise ValueError("CGT incidence anchor requires unique household ids.") - original_ids = set(household_ids[original].tolist()) - person_original = person["person_household_id"].isin(original_ids).to_numpy() - benunit_original = ( - benunit["benunit_id"] - .isin(set(person.loc[person_original, "person_benunit_id"].tolist())) - .to_numpy() - ) - multiplier = id_multiplier_for_values( - person.loc[person_original, "person_id"], - person.loc[person_original, "person_household_id"], - person.loc[person_original, "person_benunit_id"], - benunit.loc[benunit_original, "benunit_id"], - household_ids[original], - ) + multiplier = copy_id_table.radix(CGT_CLONE_COPY_ID_FIELD) position_by_id = {int(value): index for index, value in enumerate(household_ids)} clone_positions = np.flatnonzero(clone) original_positions = np.empty(len(clone_positions), dtype=np.int64) + unpaired = 0 for slot, position in enumerate(clone_positions): source = position_by_id.get(int(household_ids[position]) - multiplier) if source is None or not original[source]: - raise ValueError( - "CGT incidence anchor found a clone household without a paired " - f"original: household_id {int(household_ids[position])}." - ) + unpaired += 1 + continue original_positions[slot] = source + if unpaired: + raise ValueError( + f"CGT incidence anchor found {unpaired} clone household(s) without a " + "paired original at the cgt_incidence_clone field's radix." + ) if len(np.unique(original_positions)) != len(original_positions): raise ValueError( "CGT incidence anchor found two clone households paired to one original." @@ -808,7 +812,7 @@ def _assert_cgt_incidence_stage_parameters(stage: SourceStageSpec) -> None: "mass_split": CGT_CLONE_MASS_SPLIT, "weight_kind_out": WeightKind.IMPORTANCE.value, "conservation": "exact_total", - "id_remapping": "id_multiplier_for_values", + "id_remapping": CGT_CLONE_ID_REMAPPING, "declared_factor": 1.0, "reason": CGT_CLONE_MASS_CHANGE_REASON, }, @@ -845,12 +849,11 @@ def cgt_incidence_anchor_operation_parameters() -> tuple[ "pair_clone_households_to_originals", { "clone_flag_column": HOUSEHOLD_IS_CGT_CLONE, - "id_remapping": "id_multiplier_for_values", + "id_remapping": CGT_CLONE_ID_REMAPPING, "pairing": ( - "clone household_id minus the clone stage's id multiplier, " - "recomputed from the pre-clone rows (clone flag false; the " - "support split's copies are pre-clone rows with their own " - "clones)" + "clone household_id minus the cgt_incidence_clone field's " + "declared radix (the support split's copies are pre-clone " + "rows with their own clones)" ), "requirement": ( "bijection between clone and original households; an " diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index dc9d7c555b..8fcf89ace7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -35,6 +35,7 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_INVESTABLE_WEALTH_COLUMNS, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, @@ -61,7 +62,6 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.build.uk_runtime.spi_support import ( _exact_total_correction, _importance_weights_with_exact_total, @@ -88,6 +88,18 @@ #: A copy weighs at most this before the clone (half of it after), so the #: GBP 5m+ band seats at least a hundred rows. CGT_SUPPORT_MAXIMUM_COPY_WEIGHT = 60.0 +#: The copy-id field copy ``k`` takes ``k`` in: ordinals 1 to 9,999, so a +#: household is divided into at most 10,000 rows, root included +#: (microcosm#931). +CGT_SUPPORT_COPY_ID_FIELD = "cgt_support_split" +#: The refusal when a household needs more copies than the field holds. It is +#: fixed: it never names the household, its weight or its copy count. +CGT_SUPPORT_COPY_BOUND_REFUSAL = ( + "The CGT support split would divide a household into more copies than the " + "copy-id table's cgt_support_split field holds (ordinals up to 9,999; " + "microcosm#931). The field is declared, never sized to the frame: a " + "household this heavy needs a reviewed wider field." +) #: The Table 3 bands of gains the support is sized on. CGT_SUPPORT_MINIMUM_GAIN_BAND_LOWER = 250_000 #: The redraw's income bands, so classification matches its walk. @@ -436,7 +448,9 @@ def split_cgt_support_households( Deterministic and mass-conserving: no draw, no seed, no salt. Each selected household of weight ``w`` becomes ``n = ceil(w / 60)`` copies at ``w / n``; the root keeps its ids, copy ``k`` takes every entity id plus - ``k`` times the frame's id multiplier, records ``k`` in + ``k`` times the declared radix of the copy-id table's + ``cgt_support_split`` field (so every incoming id must sit below it, and + ``n`` is at most the field's bound of 10,000), records ``k`` in ``cgt_support_copy_index`` and carries a full copy of the household's person and benefit-unit rows. The exact-total correction lands on the heaviest unselected incumbent so every copy weight stays bitwise @@ -486,13 +500,19 @@ def split_cgt_support_households( copies[selected] = cgt_support_copy_counts( weights[selected], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT ) - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + if int(copies.max()) > copy_id_table.ordinal_bound(CGT_SUPPORT_COPY_ID_FIELD): + raise ValueError(CGT_SUPPORT_COPY_BOUND_REFUSAL) + for table, columns in ( + (person, _PERSON_ID_COLUMNS), + (benunit, ("benunit_id",)), + (household, ("household_id",)), + ): + for column in columns: + copy_id_table.require_below( + table[column], + CGT_SUPPORT_COPY_ID_FIELD, + what=f"incoming {column} values", + ) household[HOUSEHOLD_IS_CGT_SUPPORT_COPY] = False household[CGT_SUPPORT_COPIES_COLUMN] = copies household[CGT_SUPPORT_COPY_INDEX_COLUMN] = np.zeros(len(household), dtype="int64") @@ -510,14 +530,15 @@ def split_cgt_support_households( benunit["benunit_id"].isin(set(family_person["person_benunit_id"])) ].copy() family_household = household.loc[family].copy() - offset = k * multiplier for column in _PERSON_ID_COLUMNS: - family_person[column] = family_person[column].astype("int64") + offset - family_benunit["benunit_id"] = ( - family_benunit["benunit_id"].astype("int64") + offset + family_person[column] = copy_id_table.mint( + family_person[column], CGT_SUPPORT_COPY_ID_FIELD, k + ) + family_benunit["benunit_id"] = copy_id_table.mint( + family_benunit["benunit_id"], CGT_SUPPORT_COPY_ID_FIELD, k ) - family_household["household_id"] = ( - family_household["household_id"].astype("int64") + offset + family_household["household_id"] = copy_id_table.mint( + family_household["household_id"], CGT_SUPPORT_COPY_ID_FIELD, k ) family_household[HOUSEHOLD_IS_CGT_SUPPORT_COPY] = True family_household[CGT_SUPPORT_COPY_INDEX_COLUMN] = np.full( @@ -684,8 +705,9 @@ def cgt_support_split_operation_parameters() -> tuple[ "copy carries weight / copies" ), "id_remapping": ( - "id_multiplier_for_values; copy k takes every entity id + k x " - "multiplier" + "copy-id table field cgt_support_split; copy k takes every " + "entity id + k x the field's radix (10**13), k up to 9,999, " + "and a household needing more copies is refused" ), "flag_column": HOUSEHOLD_IS_CGT_SUPPORT_COPY, "copies_column": CGT_SUPPORT_COPIES_COLUMN, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py new file mode 100644 index 0000000000..5da3189426 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py @@ -0,0 +1,249 @@ +"""Fixed decimal fields for the ids of UK copy rows (microcosm#931). + +Every UK copy layer used to offset its copies' ids by a power of ten one digit +above the largest id it saw. Once the spine clones households at its start, +that largest id grows with the clone count K, so every later offset would move +with K and the K=1 spine would stop being the clone-0 slice of the K=15 spine. +Here each layer owns a decimal field of the id instead, declared up front: +copy ``k`` of a row in field ``f`` takes ``id + k x radix(f)`` on all five id +columns (``household_id``; ``person_id``, ``person_household_id`` and +``person_benunit_id``; ``benunit_id``), so every copy id is a pure function of +its source id and its copy path. + +The fields are listed least significant first, in the order the spine applies +them, each with its ordinal bound (ordinals ``0 .. bound - 1``, 0 meaning "not +a copy in this field") and its radix: + +- ``vintage``: reserved for pooled FRS vintages (the plan's P3). The spine + reads one FRS release today, so nothing mints it; raw FRS ids + (``household_id`` the sernum, ``benunit_id`` sernum x 100 + b, + ``person_id`` sernum x 1000 + p) must stay below its radix. +- ``geographic_support``: the geographic clones the clone-first spine will + mint at its start (K at most 100). Nothing mints it yet. +- ``spi_support_channel``: the SPI support copy (ordinal 1, stage 16) and the + income band donor (ordinal 2, stage 17). +- ``high_value_support``: the high-value home copies (stage 26). +- ``cgt_support_split``: the CGT support copies (stage 42), four digits. +- ``cgt_incidence_clone``: the CGT incidence twin (stage 43). +- ``cgt_residential_split``: the residential arms (stage 46), the top field, + so arms follow every incumbent in id order. + +A field must sit above every field filled before it runs, which keeps a copy's +parent below the field's radix: a copy's ordinal is then ``id // radix`` +modulo the field's width, and its parent ``id - ordinal x radix``. The table +is UK-local; a shared, country-neutral table comes with a later stage. This +module imports nothing from the UK runtime, so any module can read it. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +__all__ = [ + "UK_COPY_ID_BASE_RADIX", + "UK_COPY_ID_FIELDS", + "UK_COPY_ID_LARGEST_ID", + "CopyIdField", + "mint", + "ordinal", + "ordinal_bound", + "parent", + "radix", + "require_below", +] + +_INT64_MAX = int(np.iinfo(np.int64).max) + + +@dataclass(frozen=True) +class CopyIdField: + """One decimal field of a copy id: ordinals ``0 .. bound - 1`` at ``radix``.""" + + name: str + bound: int + radix: int + + @property + def width(self) -> int: + """The field's span, ``10 ** digits(bound - 1)``.""" + + return 10 ** len(str(self.bound - 1)) + + +#: Raw ids sit below this radix, the first field's. +UK_COPY_ID_BASE_RADIX = 10**8 +#: The fields in the order the spine applies them, least significant first. +UK_COPY_ID_FIELDS: tuple[CopyIdField, ...] = ( + CopyIdField("vintage", 10, 10**8), + CopyIdField("geographic_support", 100, 10**9), + CopyIdField("spi_support_channel", 3, 10**11), + CopyIdField("high_value_support", 6, 10**12), + CopyIdField("cgt_support_split", 10_000, 10**13), + CopyIdField("cgt_incidence_clone", 2, 10**17), + CopyIdField("cgt_residential_split", 8, 10**18), +) + + +def _largest_id(fields: Sequence[CopyIdField], base_radix: int) -> int: + """The largest id the fields can mint: every ordinal at its bound less one.""" + + return sum((field.bound - 1) * field.radix for field in fields) + base_radix - 1 + + +def _validate_fields(fields: Sequence[CopyIdField], base_radix: int) -> int: + """Refuse a malformed table; return its largest mintable id. + + The first radix is the base radix; every later radix is the previous one + times the previous field's width, so the fields are contiguous decimal + digits and none overlaps another; the largest mintable id fits int64. + """ + + names = [field.name for field in fields] + if not names or len(set(names)) != len(names): + raise ValueError("The copy-id fields must be named, each once.") + expected = base_radix + for field in fields: + if type(field.bound) is not int or field.bound < 2: + raise ValueError( + f"Copy-id field {field.name!r} needs an integer bound of at least 2." + ) + if field.radix != expected: + raise ValueError( + f"Copy-id field {field.name!r} sits at radix {field.radix}, not " + f"{expected}, so the fields are not contiguous decimal digits." + ) + expected = field.radix * field.width + largest = _largest_id(fields, base_radix) + if largest >= _INT64_MAX: + raise ValueError( + f"The copy-id fields can mint ids up to {largest}, which int64 cannot hold." + ) + return largest + + +#: About 7.2e18, below int64's 2**63 - 1. +UK_COPY_ID_LARGEST_ID = _validate_fields(UK_COPY_ID_FIELDS, UK_COPY_ID_BASE_RADIX) +_FIELDS = MappingProxyType({field.name: field for field in UK_COPY_ID_FIELDS}) + + +def _field(name: str) -> CopyIdField: + try: + return _FIELDS[name] + except KeyError as error: + raise ValueError(f"No UK copy-id field is named {name!r}.") from error + + +def radix(field: str) -> int: + """The declared radix of ``field``.""" + + return _field(field).radix + + +def ordinal_bound(field: str) -> int: + """The declared bound of ``field``: its ordinals run below it.""" + + return _field(field).bound + + +def _ids(values: object, *, what: str) -> np.ndarray: + """Exact int64 ids, refusing floats, booleans and negative ids by count.""" + + array = np.asarray(values) + if array.dtype.kind == "O": + items = array.ravel().tolist() + if any( + isinstance(item, bool | np.bool_) or not isinstance(item, int | np.integer) + for item in items + ): + raise TypeError(f"{what} must be integers.") + if any(not -_INT64_MAX - 1 <= int(item) <= _INT64_MAX for item in items): + raise ValueError(f"{what} must fit int64.") + array = np.asarray([int(item) for item in items], dtype=np.int64).reshape( + array.shape + ) + elif array.dtype.kind == "u": + if array.size and int(array.max()) > _INT64_MAX: + raise ValueError(f"{what} must fit int64.") + elif array.dtype.kind != "i": + raise TypeError(f"{what} must be integers, not {array.dtype}.") + result = array.astype(np.int64, copy=False) + negative = int((result < 0).sum()) + if negative: + raise ValueError(f"{negative} of the {what} are negative.") + return result + + +def require_below(ids: object, field: str, *, what: str = "ids") -> None: + """Refuse any id at or above ``field``'s radix, printing only the count. + + A stage that mints ``field`` copies rows whose ids must all sit below it, + so that every copy's parent is recoverable from the copy's id. + """ + + values = _ids(ids, what=what) + reached = int((values >= radix(field)).sum()) + if reached: + raise ValueError( + f"{reached} of the {what} reach the {field!r} copy-id field " + f"(radix {radix(field)}), which only a later stage may fill " + "(microcosm#931)." + ) + + +def ordinal(ids: object, field: str) -> np.ndarray: + """Each id's ordinal in ``field``: ``(id // radix) % width``.""" + + spec = _field(field) + return (_ids(ids, what="ids") // spec.radix) % spec.width + + +def parent(ids: object, field: str) -> np.ndarray: + """Each id with its ``field`` digits cleared: the row it was copied from.""" + + values = _ids(ids, what="ids") + spec = _field(field) + return values - ((values // spec.radix) % spec.width) * spec.radix + + +def mint(ids: object, field: str, ordinals: object) -> np.ndarray: + """``ids + ordinals x radix(field)`` as int64, refusing every malformed copy. + + Refuses an ordinal outside ``[1, bound)`` in a fixed sentence (it names + the field and its declared bound, never how many copies a row needed), + an id whose ``field`` digits are already set (a copy of a copy in the same + field), and a result int64 cannot hold. ``ordinals`` is one integer or one + per id. The other messages carry counts and declared constants only, + never an id. + """ + + spec = _field(field) + values = _ids(ids, what="ids") + steps = _ids(ordinals, what=f"{field!r} ordinals") + try: + steps = np.broadcast_to(steps, values.shape) + except ValueError as error: + raise ValueError( + f"{field!r} ordinals must be one integer or one per id." + ) from error + if ((steps < 1) | (steps >= spec.bound)).any(): + raise ValueError( + f"A {field!r} copy ordinal falls outside [1, {spec.bound}), the " + "field's declared bound (microcosm#931)." + ) + occupied = int((((values // spec.radix) % spec.width) != 0).sum()) + if occupied: + raise ValueError( + f"{occupied} id(s) already carry a {field!r} ordinal; a copy is minted " + "from a row that is not a copy in its own field (microcosm#931)." + ) + offsets = steps * spec.radix + overflow = int((values > _INT64_MAX - offsets).sum()) + if overflow: + raise ValueError( + f"{overflow} {field!r} copy id(s) would overflow int64 (microcosm#931)." + ) + return values + offsets diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 11a503d900..a0bcf9cb12 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -13,6 +13,7 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.trace import sha256_file +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.birth_month import ( UK_BIRTH_MONTH_COLUMN, uk_months_since_last_birthday, @@ -677,9 +678,32 @@ def build_uk_frs_spine_frame( normalized = {name: _normalize_ids(table) for name, table in tables.items()} frame = _assemble_frame(normalized, evidence=evidence) validate_uk_national_frame(frame) + _require_raw_ids_below_copy_fields(frame) return frame +def _require_raw_ids_below_copy_fields(frame: Frame) -> None: + """Refuse raw ids that reach the first copy-id field (microcosm#931). + + Every copy layer adds its ordinal times a declared radix to its copies' + ids, the lowest radix being the reserved ``vintage`` field's, so a raw + id (person_id is sernum x 1000 + p) must sit below it. + """ + + person = frame.table("person") + for entity, column in ( + ("person", "person_id"), + ("person", "person_household_id"), + ("person", "person_benunit_id"), + ("benunit", "benunit_id"), + ("household", "household_id"), + ): + table = person if entity == "person" else frame.table(entity) + copy_id_table.require_below( + table[column], "vintage", what=f"raw FRS {column} values" + ) + + def _artifact_by_table(stage: SourceStageSpec) -> dict[str, Mapping[str, Any]]: by_table: dict[str, Mapping[str, Any]] = {} for artifact in stage.artifacts: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index f52bd8f413..2d207dac9a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -55,6 +55,8 @@ from . import ( bus_use_incidence, cgt_imputation, + cgt_support, + copy_id_table, energy_pricing, frs_disability, frs_education, @@ -82,7 +84,6 @@ was_wealth, ) from .national_frame import UK_NATIONAL_SCHEMA -from .rowwise_geography import id_multiplier_for_values if TYPE_CHECKING: from microcosm.build.source_manifest import SourceStageSpec @@ -186,7 +187,20 @@ # in spi_income, so an edit there moves the stage's hash (microcosm#1278), # and its twin transport in spi_rank_transport (microcosm#840). "hmrc_spi_income_spine": (spi_income, spi_rank_transport), - "frs_spine": (uc_relationships,), + # The spine's raw ids sit below the copy-id table's first field, and every + # copy layer mints its ids in its own field of the table; the two SPI + # support stages mint through the support channel's clone helper, and the + # high-value split weighs its copies through the CGT split's exact-total + # correction (microcosm#931). The executor reads each id-offset stage's + # declared radix from the table too. + "frs_spine": (uc_relationships, copy_id_table), + "spi_support_channel": (spi_support, copy_id_table), + "spi_income_band_donors": (spi_support, copy_id_table), + "high_value_home_support": (cgt_support, copy_id_table), + "cgt_support_split": (copy_id_table,), + "cgt_incidence_clone": (rowwise_geography, copy_id_table), + "cgt_incidence_anchor": (rowwise_geography, copy_id_table), + "cgt_residential_split": (copy_id_table,), "frs_legacy_proxies": (*_ENGINE_MODULES,), "frs_education_grant_split": (*_ENGINE_MODULES,), "frs_brma": (*_ENGINE_MODULES,), @@ -1336,6 +1350,56 @@ def _expand_cells(context: KernelContext) -> tuple[tuple[str, str, str], ...]: return tuple(tuple(str(part) for part in item) for item in raw) # type: ignore[arg-type] +#: The copy-id field each id-offset structural stage mints (microcosm#931): +#: copy ``k`` takes ``source + k x radix(field)`` on every id column, and +#: every incumbent id lies below the field's radix, so the copy index is the +#: integer quotient. The SPI support stages name each copy's source in +#: ``{entity}_source_id`` instead. +_COPY_ID_FIELDS = MappingProxyType( + { + "high_value_home_support": "high_value_support", + "cgt_support_split": "cgt_support_split", + "cgt_incidence_clone": "cgt_incidence_clone", + "cgt_residential_split": "cgt_residential_split", + } +) + + +def _declared_copy_radix(stage: str, before: Frame) -> int | None: + """The declared radix an id-offset stage mints at, or None for the others. + + Every incumbent id and membership must sit below it, or the quotient + would name the wrong parent; the refusal prints counts only. + """ + + field = _COPY_ID_FIELDS.get(stage) + if field is None: + return None + person = before.table(before.schema.person_entity) + columns = [ + (before.table(entity), before.schema.entity_id_column(entity)) + for entity in before.entities + ] + [ + (person, before.schema.membership_column(group)) + for group in before.schema.group_entities + ] + for table, column in columns: + copy_id_table.require_below( + table[column], + field, + what=f"UK stage {stage!r} incumbent {column} values", + ) + return copy_id_table.radix(field) + + +def _exact_ids(values: pd.Series, *, label: str) -> np.ndarray: + """An id column as exact int64: ids reach 2**53 and above (microcosm#931).""" + + if not pd.api.types.is_integer_dtype(values.dtype): + raise TypeError(f"{label} must be an integer id column, not {values.dtype}.") + return values.to_numpy(dtype=np.int64) + + def _source_lineage( before: Frame, after: Frame, @@ -1343,62 +1407,64 @@ def _source_lineage( *, id_offset: int | None, ) -> pd.Series: - """Derive new target-to-immediate-source ids from a structural result.""" + """Derive new target-to-immediate-source ids from a structural result. + + Vectorised on exact int64 ids: a row-by-row walk would upcast an + all-numeric row to float64, which is inexact above 2**53. The refusals + print counts, never an id. + """ id_column = before.schema.entity_id_column(entity) before_table = before.table(entity) after_table = after.table(entity) + dtype = before_table[id_column].dtype before_ids = pd.Index(before_table[id_column]) + after_ids = _exact_ids(after_table[id_column], label=f"{entity}.{id_column}") + new = ~pd.Index(after_ids).isin(before_ids) # A weights-only structural stage (the #970 incidence anchor) adds no - # row: every target is an incumbent, so the lineage is empty without - # walking the table row by row. - if pd.Index(after_table[id_column]).isin(before_ids).all(): + # row: every target is an incumbent, so the lineage is empty. + if not new.any(): return pd.Series( [], - index=pd.Index([], name=id_column, dtype=before_table[id_column].dtype), - dtype=before_table[id_column].dtype, + index=pd.Index([], name=id_column, dtype=dtype), + dtype=dtype, name=id_column, ) - targets: list[object] = [] - values: list[object] = [] + targets = after_ids[new] source_column = f"{entity}_source_id" - for _, row in after_table.iterrows(): - target = row[id_column] - if target in before_ids: - continue - - # CGT stages retain long-lived source-id provenance from the SPI - # support stage, so their immediate clone lineage is the stage's - # reviewed ID offset, not that older provenance column. Copy ``k`` - # of a CGT structural stage takes ``source + k * offset`` (the clone - # stage mints one copy, the support split up to ``ceil(w / cap)``, - # microcosm#1045), and every incumbent id lies below the offset, so - # the copy index is the integer quotient. - if id_offset is not None: - copy_index = int(target) // int(id_offset) - candidate = target - copy_index * id_offset - if copy_index >= 1 and candidate in before_ids: - targets.append(target) - values.append(candidate) - continue + if id_offset is not None: + # CGT and high-value stages retain long-lived source-id provenance + # from the SPI support stage, so their immediate lineage is the + # stage's declared copy-id radix, not that older provenance column. + copy_index = targets // int(id_offset) + sources = targets - copy_index * int(id_offset) + unmatched = int(((copy_index < 1) | ~pd.Index(sources).isin(before_ids)).sum()) + if unmatched: raise ValueError( - f"UK stage produced {entity!r} target id {target!r} whose " - f"offset lineage {candidate!r} is not an incumbent id." + f"UK stage produced {unmatched} new {entity!r} row(s) whose id " + "less its copy ordinal times the stage's declared copy-id radix " + "is not an incumbent id." ) - - if source_column in after_table and row[source_column] in before_ids: - targets.append(target) - values.append(row[source_column]) - continue - + elif source_column in after_table: + # Only the new rows' sources are read, as the row walk read them. + sources = _exact_ids( + after_table[source_column][new], label=f"{entity}.{source_column}" + ) + unmatched = int((~pd.Index(sources).isin(before_ids)).sum()) + if unmatched: + raise ValueError( + f"UK stage produced {unmatched} new {entity!r} row(s) whose " + f"{source_column!r} is not an incumbent id." + ) + else: raise ValueError( - f"UK stage produced {entity!r} target id {target!r} without an " - f"explicit {source_column!r} or a declared ID-offset lineage rule." + f"UK stage produced {int(new.sum())} new {entity!r} row(s) without an " + f"explicit {source_column!r} or a declared copy-id lineage rule." ) return pd.Series( - values, - index=pd.Index(targets, name=id_column, dtype=before_table[id_column].dtype), - dtype=before_table[id_column].dtype, + sources, + index=pd.Index(targets, name=id_column, dtype=dtype), + dtype=dtype, name=id_column, ) @@ -1444,25 +1510,7 @@ def run(self, context: KernelContext) -> KernelResult: f"UK stage {self.stage!r} returned {type(after).__name__}, not Frame." ) cells = _expand_cells(context) - id_offset = None - if self.stage in { - "high_value_home_support", - "cgt_support_split", - "cgt_incidence_clone", - "cgt_residential_split", - }: - id_offset = id_multiplier_for_values( - *( - before.table(entity)[before.schema.entity_id_column(entity)] - for entity in before.entities - ), - *( - before.table(before.schema.person_entity)[ - before.schema.membership_column(group) - ] - for group in before.schema.group_entities - ), - ) + id_offset = _declared_copy_radix(self.stage, before) expand: dict[str, pd.Series] = {} for entity in before.entities: expand[entity] = _source_lineage( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index 8c354334b1..fa08b8beb5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -314,8 +314,13 @@ def run(self, context: KernelContext) -> KernelResult: table = after.table(entity) id_column = before.schema.entity_id_column(entity) added = table.iloc[before.n(entity) :] + # One clone adds no row. A spine whose copy ids reach the copy-id + # table's top fields (microcosm#931) has a multiplier above int64, + # and multiplying even an empty int64 column by it raises. source_ids = ( - added[id_column].to_numpy() + np.empty(0, dtype=np.int64) + if added.empty + else added[id_column].to_numpy() - added[ladder_clone_index_column(entity)].to_numpy() * pool.id_multiplier ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py index 4459b95625..a898110e10 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py @@ -31,6 +31,7 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations from microcosm.build.uk_runtime.cgt_support import _exact_support_weights from microcosm.build.uk_runtime.household_lineage import ( @@ -46,12 +47,12 @@ SUPPORT_STRATUM_LOWER, owner_mask, ) -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.frame import Frame, MassChangeRecord, WeightKind __all__ = [ "HIGH_VALUE_SUPPORT_COPIES", "HIGH_VALUE_SUPPORT_COPIES_COLUMN", + "HIGH_VALUE_SUPPORT_COPY_ID_FIELD", "HIGH_VALUE_SUPPORT_MASS_CHANGE_REASON", "HIGH_VALUE_SUPPORT_OPERATION_KIND", "HIGH_VALUE_SUPPORT_STAGE_NAME", @@ -73,6 +74,8 @@ HIGH_VALUE_SUPPORT_STRATUM_LOWER = SUPPORT_STRATUM_LOWER #: Copies per stratum household, root included (ruled six). HIGH_VALUE_SUPPORT_COPIES = 6 +#: The copy-id field copy ``k`` takes ``k`` in (microcosm#931). +HIGH_VALUE_SUPPORT_COPY_ID_FIELD = "high_value_support" HIGH_VALUE_SUPPORT_MASS_CHANGE_REASON = ( "High-value home support divides every owner household in the top value " "stratum of its region into light copies at equal weight; every household's " @@ -145,7 +148,8 @@ def high_value_support_operation_parameters() -> tuple[ ), "copy_rule": "every selected household becomes copies rows at weight / copies", "id_remapping": ( - "id_multiplier_for_values; copy k takes every entity id + k x multiplier" + "copy-id table field high_value_support; copy k takes every " + "entity id + k x the field's radix (10**12)" ), "flag_column": HIGH_VALUE_SUPPORT_COPY_COLUMN, "copies_column": HIGH_VALUE_SUPPORT_COPIES_COLUMN, @@ -235,11 +239,12 @@ def split_high_value_support_households( Each selected household of weight ``w`` becomes ``copies`` rows at ``w / copies``; the root keeps its ids, copy ``k`` takes every entity id - plus ``k`` times the frame's id multiplier, records ``k`` in - ``high_value_support_copy_index`` and carries a full copy of its person - and benefit-unit rows. The exact-total correction lands on the heaviest - unselected incumbent so every copy weight stays bitwise ``w / copies``. - Refuses a frame this stage has already split. + plus ``k`` times the declared radix of the copy-id table's + ``high_value_support`` field (so every incoming id must sit below it), + records ``k`` in ``high_value_support_copy_index`` and carries a full copy + of its person and benefit-unit rows. The exact-total correction lands on + the heaviest unselected incumbent so every copy weight stays bitwise + ``w / copies``. Refuses a frame this stage has already split. """ if isinstance(copies, bool) or copies < 2: raise ValueError("High-value support copies must be an integer of at least 2.") @@ -275,13 +280,17 @@ def split_high_value_support_households( ) copy_counts = np.ones(len(household), dtype="int64") copy_counts[selected] = int(copies) - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + for table, columns in ( + (person, _PERSON_ID_COLUMNS), + (benunit, ("benunit_id",)), + (household, ("household_id",)), + ): + for column in columns: + copy_id_table.require_below( + table[column], + HIGH_VALUE_SUPPORT_COPY_ID_FIELD, + what=f"incoming {column} values", + ) household[HIGH_VALUE_SUPPORT_COPY_COLUMN] = False household[HIGH_VALUE_SUPPORT_COPIES_COLUMN] = copy_counts household[HIGH_VALUE_SUPPORT_INDEX_COLUMN] = np.zeros(len(household), dtype="int64") @@ -299,14 +308,15 @@ def split_high_value_support_households( benunit["benunit_id"].isin(set(family_person["person_benunit_id"])) ].copy() family_household = household.loc[family].copy() - offset = k * multiplier for column in _PERSON_ID_COLUMNS: - family_person[column] = family_person[column].astype("int64") + offset - family_benunit["benunit_id"] = ( - family_benunit["benunit_id"].astype("int64") + offset + family_person[column] = copy_id_table.mint( + family_person[column], HIGH_VALUE_SUPPORT_COPY_ID_FIELD, k + ) + family_benunit["benunit_id"] = copy_id_table.mint( + family_benunit["benunit_id"], HIGH_VALUE_SUPPORT_COPY_ID_FIELD, k ) - family_household["household_id"] = ( - family_household["household_id"].astype("int64") + offset + family_household["household_id"] = copy_id_table.mint( + family_household["household_id"], HIGH_VALUE_SUPPORT_COPY_ID_FIELD, k ) family_household[HIGH_VALUE_SUPPORT_COPY_COLUMN] = True family_household[HIGH_VALUE_SUPPORT_INDEX_COLUMN] = np.full( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index 603927459a..d81a85fad0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -88,6 +88,45 @@ #: the legacy single-year schema, unchanged by the Frame carrier). ARTIFACT_CLONE_INDEX_COLUMN = "clone_index" +_INT64_MAX = int(np.iinfo(np.int64).max) + + +def refuse_dense_clone_id_overflow( + *, + n_clones: int, + id_multiplier: int, + ids: tuple[object, ...], +) -> None: + """Refuse dense clones whose ids int64 cannot hold, in one clear message. + + Dense clone ``c`` offsets every id by ``c`` times a multiplier above the + spine's largest id. Since microcosm#931 the spine mints its copies' ids in + fixed fields of the copy-id table, up to the CGT incidence twins at + 10**17 and the residential arms at 10**18, so a production spine leaves no + room for a second clone above its ids. Dense builds with more than one + clone wait for the clone-first spine, which mints the geographic clones at + its start (#931 group (c)); one clone, and a spine without copy rows, + expand as before. The message names no id. + """ + + if type(n_clones) is not int or n_clones <= 1: + return + largest = 0 + for values in ids: + array = np.asarray(values) + if array.size: + largest = max(largest, int(np.abs(array.astype(np.int64)).max())) + if (n_clones - 1) * int(id_multiplier) + largest > _INT64_MAX: + raise ValueError( + f"Dense geographic clones of this spine at n_clones={n_clones} would " + "overflow int64: since microcosm#931 the spine mints its copy ids in " + "the copy-id table's fixed fields, up to 10**18, and every dense " + "clone is offset above the spine's largest id. Dense builds with " + "more than one clone wait for the clone-first spine of microcosm#931, " + "which mints the geographic clones at its start; n_clones=1 still " + "expands." + ) + def ladder_clone_index_column(entity: str) -> str: """The in-memory, per-entity clone-index column on the ladder frame. @@ -226,6 +265,15 @@ def clone_uk_dataset_tables_with_rowwise_geography( person_frame["person_benunit_id"], benunit_frame["benunit_id"], ) + refuse_dense_clone_id_overflow( + n_clones=n_clones, + id_multiplier=id_multiplier, + ids=( + household_frame[household_id_column], + *(person_frame[column] for column in PERSON_ID_COLUMNS), + benunit_frame["benunit_id"], + ), + ) assignment = assign_household_geography( household_frame, crosswalk, @@ -415,6 +463,15 @@ def expand_uk_geographic_pool( person_frame["person_benunit_id"], benunit_frame["benunit_id"], ) + refuse_dense_clone_id_overflow( + n_clones=n_clones, + id_multiplier=id_multiplier, + ids=( + household_frame["household_id"], + *(person_frame[column] for column in PERSON_ID_COLUMNS), + benunit_frame["benunit_id"], + ), + ) cloned_household = clone_entity_frame( household_frame, id_columns=HOUSEHOLD_ID_COLUMNS, @@ -1125,10 +1182,12 @@ def apply_uk_source_lineage_modulus( layer*, leaving any immediate-layer ``source_household_id`` or ``source_household_key`` untouched. - Spine inputs use a different scheme: ``household_id = sernum + - 1{spi} * 10**d + k{support_copy} * 10**(d+1) + 1{cgt_clone} * 10**(d+2)``. - Their explicit spine lineage columns are the source of truth, so applying - the pool modulus would mint a false second lineage layer and is refused. + Spine inputs use a different scheme, the copy-id table's fixed fields + (microcosm#931): ``household_id = sernum + ordinal{spi} * 10**11 + + k{high_value_copy} * 10**12 + k{support_copy} * 10**13 + + 1{cgt_clone} * 10**17 + j{residential_arm} * 10**18``. Their explicit spine + lineage columns are the source of truth, so applying the pool modulus + would mint a false second lineage layer and is refused. The mapping is refused when it would be ambiguous (the pool column already exists) or vacuous (no id reaches the modulus, making it the diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py index 3b04d354a1..36cb38b6ad 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py @@ -42,7 +42,6 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.build.uk_runtime.spi_income import ( SPI_MINIMUM_RECIPIENT_AGE, SPI_RECIPIENT_ROLE_COLUMN, @@ -62,6 +61,7 @@ _clone_support_frame, _exact_total_correction, support_channel_column, + support_source_id_column, ) from microcosm.frame import Frame, MassChangeRecord, WeightKind, engine_tables @@ -101,8 +101,15 @@ "draw_spi_donor_ages_by_population" ) SPI_INCOME_BAND_DONOR_DRAW_SALT = "spi_income_band_donor_draw" -#: The support channel copies at clone index 1; the reserved copies sit at 2. +#: The support channel copies at clone index 1; the reserved copies sit at 2, +#: the ordinal they take in the copy-id table's spi_support_channel field. SPI_INCOME_BAND_DONOR_CLONE_INDEX = 2 +#: The id rule the stage manifest restates (microcosm#931). +SPI_INCOME_BAND_DONOR_ID_REMAPPING = ( + "copy-id table field spi_support_channel; a donor takes every entity id + " + "2 x the field's radix (10**11), and its source is read from its source-id " + "columns, never from the id" +) HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR = "household_is_spi_income_band_donor" SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN = "spi_income_band_donor_lower_bound" PERSON_IS_SPI_INCOME_BAND_CARRIER = "person_is_spi_income_band_carrier" @@ -620,20 +627,15 @@ def stack_spi_income_band_donors( ) selected_ids = set(selected_household) - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + # The donors are copies of base households, minted at ordinal 2 of the + # copy-id table's spi_support_channel field; each donor's source is the + # {entity}_source_id the copy carries (microcosm#931). donor_household = _clone_support_frame( household.loc[household["household_id"].isin(selected_ids)], entity="household", id_columns=_HOUSEHOLD_ID_COLUMNS, channel=SPI_SYNTHETIC_SUPPORT_CHANNEL, clone_index=SPI_INCOME_BAND_DONOR_CLONE_INDEX, - id_multiplier=multiplier, ) donor_person = _clone_support_frame( person.loc[person["person_household_id"].isin(selected_ids)], @@ -641,7 +643,6 @@ def stack_spi_income_band_donors( id_columns=_PERSON_ID_COLUMNS, channel=SPI_SYNTHETIC_SUPPORT_CHANNEL, clone_index=SPI_INCOME_BAND_DONOR_CLONE_INDEX, - id_multiplier=multiplier, ) selected_benunits = set( person.loc[ @@ -654,10 +655,9 @@ def stack_spi_income_band_donors( id_columns=_BENUNIT_ID_COLUMNS, channel=SPI_SYNTHETIC_SUPPORT_CHANNEL, clone_index=SPI_INCOME_BAND_DONOR_CLONE_INDEX, - id_multiplier=multiplier, ) - source_household = donor_household["household_id"].to_numpy() - ( - SPI_INCOME_BAND_DONOR_CLONE_INDEX * multiplier + source_household = donor_household[support_source_id_column("household")].to_numpy( + dtype="int64" ) donor_band = np.asarray( [selected_household[int(value)] for value in source_household], dtype=float @@ -668,8 +668,8 @@ def stack_spi_income_band_donors( donor_household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR] = True donor_household[SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN] = donor_band person[PERSON_IS_SPI_INCOME_BAND_CARRIER] = False - source_person = donor_person["person_id"].to_numpy() - ( - SPI_INCOME_BAND_DONOR_CLONE_INDEX * multiplier + source_person = donor_person[support_source_id_column("person")].to_numpy( + dtype="int64" ) donor_person[PERSON_IS_SPI_INCOME_BAND_CARRIER] = np.isin( source_person, np.asarray(list(carrier_person), dtype="int64") @@ -994,6 +994,7 @@ def spi_income_band_donor_operation_parameters( "seed": seed, "salt": SPI_INCOME_BAND_DONOR_DRAW_SALT, "clone_index": SPI_INCOME_BAND_DONOR_CLONE_INDEX, + "id_remapping": SPI_INCOME_BAND_DONOR_ID_REMAPPING, "flag_column": HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, "band_column": SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN, "carrier_column": PERSON_IS_SPI_INCOME_BAND_CARRIER, @@ -1047,6 +1048,7 @@ def _assert_band_donor_stage_parameters(stage: SourceStageSpec, *, seed: int) -> "PERSON_IS_SPI_INCOME_BAND_CARRIER", "SPI_INCOME_BAND_DONOR_CLONE_INDEX", "SPI_INCOME_BAND_DONOR_DRAW_SALT", + "SPI_INCOME_BAND_DONOR_ID_REMAPPING", "SPI_INCOME_BAND_DONOR_LOWER_BOUNDS", "SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN", "SPI_INCOME_BAND_DONOR_MASS_CHANGE_REASON", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index dbdc002611..3b40189de6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -119,6 +119,7 @@ SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, SPI_INCOME_QRF_OUTPUT_COLUMNS, + SPI_SUPPORT_ID_REMAPPING, SPI_SYNTHETIC_SUPPORT_CHANNEL, UKSPISupportResult, build_uk_spi_support_channel, @@ -1190,6 +1191,11 @@ def _support_stage_parameters( raise ValueError("SPI support manifest seed drifted from the reviewed value.") if stack.parameters.get("draw") != "uniform_without_replacement": raise ValueError("SPI support manifest must declare a uniform donor draw.") + if stack.parameters.get("id_remapping") != SPI_SUPPORT_ID_REMAPPING: + raise ValueError( + "SPI support manifest id_remapping drifted from the copy-id table's " + "spi_support_channel rule." + ) if stack.parameters.get("flag_column") != HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: raise ValueError("SPI support manifest flag column drifted.") if dict(stack.parameters.get("channels", {})) != SPI_SPINE_SUPPORT_CHANNELS: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py index 8c8d7ed86b..7b21e65380 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py @@ -16,10 +16,10 @@ import numpy as np import pandas as pd +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.hmrc_income import ( HMRC_SPI_ASSESSABLE_INCOME_COLUMN, ) -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.build.uk_runtime.terminal_gates import ( UKZeroWeightStratumDeclaration, uk_zero_weight_strata_gate, @@ -33,6 +33,14 @@ BASE_FRS_SUPPORT_CHANNEL = "frs" SPI_SYNTHETIC_SUPPORT_CHANNEL = "spi" UK_SPI_SUPPORT_STAGE_NAME = "spi_support_channel" +#: The copy-id field the SPI copies (ordinal 1) and the income band donors +#: (ordinal 2) take (microcosm#931). +SPI_SUPPORT_COPY_ID_FIELD = "spi_support_channel" +#: The id rule the stage manifest restates. +SPI_SUPPORT_ID_REMAPPING = ( + "copy-id table field spi_support_channel; the SPI copy takes every entity " + "id + 1 x the field's radix (10**11)" +) DEFAULT_SPI_SUPPORT_HOUSEHOLDS = 10_000 DEFAULT_SPI_PRIOR_MASS_SHARE = 0.5 HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN = "household_is_spi_synthetic" @@ -257,7 +265,6 @@ def create_uk_spi_support_tables( spi_household_count: int | None = DEFAULT_SPI_SUPPORT_HOUSEHOLDS, seed: int = 42, source_year: int | None = None, - id_multiplier: int | None = None, selected_household_ids: Sequence[Any] | None = None, ) -> UKSPISupportResult: """Create a zero-weight SPI support copy from UK single-year tables. @@ -268,6 +275,10 @@ def create_uk_spi_support_tables( columns that deliberately point back to the original FRS household. Keeping that lineage stable prevents long local-geography support summaries from counting the SPI copy as additional independent FRS support. + + Each copy's five id columns take ``id + radix``, the declared radix of the + copy-id table's ``spi_support_channel`` field (:data:`SPI_SUPPORT_ID_REMAPPING`), + so every incoming id must sit below it (microcosm#931). """ person_frame = person.copy() @@ -279,16 +290,16 @@ def create_uk_spi_support_tables( _validate_uk_support_inputs(person_frame, benunit_frame, household_frame) _reject_metadata_collisions(person_frame, benunit_frame, household_frame) - if id_multiplier is None: - id_multiplier = id_multiplier_for_values( - household_frame["household_id"], - person_frame["person_id"], - person_frame["person_household_id"], - person_frame["person_benunit_id"], - benunit_frame["benunit_id"], + for column, values in ( + ("household_id", household_frame["household_id"]), + ("person_id", person_frame["person_id"]), + ("person_household_id", person_frame["person_household_id"]), + ("person_benunit_id", person_frame["person_benunit_id"]), + ("benunit_id", benunit_frame["benunit_id"]), + ): + copy_id_table.require_below( + values, SPI_SUPPORT_COPY_ID_FIELD, what=f"incoming {column} values" ) - elif id_multiplier <= 0: - raise ValueError("id_multiplier must be positive.") if selected_household_ids is None: selected_household_ids = _sample_spi_household_ids( @@ -316,7 +327,6 @@ def create_uk_spi_support_tables( id_columns=_HOUSEHOLD_ID_COLUMNS, channel=BASE_FRS_SUPPORT_CHANNEL, clone_index=0, - id_multiplier=id_multiplier, ) base_household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN] = False @@ -326,7 +336,6 @@ def create_uk_spi_support_tables( id_columns=_HOUSEHOLD_ID_COLUMNS, channel=SPI_SYNTHETIC_SUPPORT_CHANNEL, clone_index=1, - id_multiplier=id_multiplier, ) spi_household["household_weight"] = 0.0 spi_household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN] = True @@ -337,7 +346,6 @@ def create_uk_spi_support_tables( id_columns=_PERSON_ID_COLUMNS, channel=BASE_FRS_SUPPORT_CHANNEL, clone_index=0, - id_multiplier=id_multiplier, ) spi_person = _clone_support_frame( selected_person, @@ -345,7 +353,6 @@ def create_uk_spi_support_tables( id_columns=_PERSON_ID_COLUMNS, channel=SPI_SYNTHETIC_SUPPORT_CHANNEL, clone_index=1, - id_multiplier=id_multiplier, ) base_benunit = _clone_support_frame( @@ -354,7 +361,6 @@ def create_uk_spi_support_tables( id_columns=_BENUNIT_ID_COLUMNS, channel=BASE_FRS_SUPPORT_CHANNEL, clone_index=0, - id_multiplier=id_multiplier, ) spi_benunit = _clone_support_frame( selected_benunit, @@ -362,14 +368,13 @@ def create_uk_spi_support_tables( id_columns=_BENUNIT_ID_COLUMNS, channel=SPI_SYNTHETIC_SUPPORT_CHANNEL, clone_index=1, - id_multiplier=id_multiplier, ) result = UKSPISupportResult( person=pd.concat([base_person, spi_person], ignore_index=True), benunit=pd.concat([base_benunit, spi_benunit], ignore_index=True), household=pd.concat([base_household, spi_household], ignore_index=True), - id_multiplier=id_multiplier, + id_multiplier=copy_id_table.radix(SPI_SUPPORT_COPY_ID_FIELD), spi_household_ids=selected_household_ids, ) _validate_uk_support_outputs(result.person, result.benunit, result.household) @@ -401,6 +406,13 @@ def replace_uk_spi_support_tables( weights to ``IMPORTANCE`` and records a deliberate, factor-one :class:`MassChangeRecord`; national and stratum mass never double or disappear. + + The rebuilt copies mint their ids in the copy-id table's + ``spi_support_channel`` field, as the spine stage does, so a base whose + surviving ids already reach that field (a compact whose ids carry other + copy or clone offsets) is refused by count rather than given colliding + ids (microcosm#931). No build calls this path; its tests run on + synthetic bases. """ if not isinstance(input_weight_kind, WeightKind): @@ -734,19 +746,22 @@ def _clone_support_frame( id_columns: tuple[str, ...], channel: str, clone_index: int, - id_multiplier: int, ) -> pd.DataFrame: + """Copy rows into a support channel; clone ``k >= 1`` mints its ids. + + The copy's id columns take ``id + k x radix`` in the copy-id table's + ``spi_support_channel`` field (the SPI copy at 1, the income band donor at + 2), which refuses an ordinal outside the field, an id already carrying + one and an int64 overflow; ``{entity}_source_id`` keeps the row's own id. + """ + clone = frame.copy() source_id = support_source_id_column(entity) clone[source_id] = clone[_entity_id_column(entity)].to_numpy() clone[support_channel_column(entity)] = channel clone[support_clone_index_column(entity)] = clone_index for column in id_columns: - clone[column] = _remap_ids( - clone[column].to_numpy(), - clone_index=clone_index, - id_multiplier=id_multiplier, - ) + clone[column] = _remap_ids(clone[column].to_numpy(), clone_index=clone_index) return clone @@ -1274,16 +1289,11 @@ def _source_household_keys( return keys -def _remap_ids( - ids: Sequence[Any], - *, - clone_index: int, - id_multiplier: int, -) -> np.ndarray: +def _remap_ids(ids: Sequence[Any], *, clone_index: int) -> np.ndarray: values = pd.to_numeric(pd.Series(ids), errors="raise").astype("int64").to_numpy() if clone_index == 0: return values.copy() - return values + clone_index * id_multiplier + return copy_id_table.mint(values, SPI_SUPPORT_COPY_ID_FIELD, clone_index) def _entity_id_column(entity: str) -> str: @@ -1335,6 +1345,8 @@ def _require_entity(entity: str) -> str: "SPI_PROFIT_BEFORE_LOSSES_DRAW_COLUMN", "SPI_PROPERTY_FINANCE_COSTS_COLUMN", "SPI_REPLACEMENT_STRATA_COLUMNS", + "SPI_SUPPORT_COPY_ID_FIELD", + "SPI_SUPPORT_ID_REMAPPING", "UKSPISupportResult", "UK_SPI_SUPPORT_STAGE_NAME", "build_uk_spi_support_channel", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_residential_split.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_residential_split.py index f36cbf30d3..57bb7f5dbe 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_residential_split.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_residential_split.py @@ -35,8 +35,12 @@ residential_arm_factors, split_cgt_residential_households, ) +from microcosm.build.uk_runtime.copy_id_table import radix from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.frame import WeightKind +from test_support.microcosm_build.uk_clone_first_frames import ( + assert_copy_ids_follow_lineage, +) PARAMETERS = UKCGTPolicyParameters( personal_allowance=12_570.0, @@ -178,13 +182,10 @@ def test_split_carries_table_8a_as_identities_and_conserves_mass_bitwise() -> No liable = gains > PARAMETERS.annual_exempt_amount assert int(arms.sum()) == int(liable.sum()) == evidence["arms_created"] assert evidence["households_by_liable_gainers"] == {"1": int(liable.sum())} - multiplier = cgt_residential_split.id_multiplier_for_values( - frame.table("person")["person_id"], - frame.table("person")["person_household_id"], - frame.table("person")["person_benunit_id"], - frame.table("benunit")["benunit_id"], - frame.table("household")["household_id"], - ) + # Arm j sits at j times the copy-id table's top-field radix + # (microcosm#931), above every incoming id. + multiplier = radix(cgt_residential_split.CGT_RESIDENTIAL_COPY_ID_FIELD) + assert multiplier == 10**18 probability = person.set_index("person_id")[CGT_RESIDENTIAL_PROBABILITY_COLUMN] for source in np.flatnonzero(liable): p = float(probability.loc[source]) @@ -225,16 +226,15 @@ def test_households_with_several_liable_gainers_become_product_arms() -> None: ) probability = person.set_index("person_id")[CGT_RESIDENTIAL_PROBABILITY_COLUMN] p0, p1 = float(probability.loc[0]), float(probability.loc[1]) - multiplier = 10 + multiplier = radix("cgt_residential_split") assert weights.loc[0] == pytest.approx(10.0 * (1 - p0) * (1 - p1)) assert weights.loc[0 + multiplier] == pytest.approx(10.0 * p0 * (1 - p1)) assert weights.loc[0 + 2 * multiplier] == pytest.approx(10.0 * (1 - p0) * p1) assert weights.loc[0 + 3 * multiplier] == pytest.approx(10.0 * p0 * p1) - assert household.loc[[10, 20, 30], CGT_RESIDENTIAL_CLONE_INDEX_COLUMN].tolist() == [ - 1, - 2, - 3, - ] + assert household.loc[ + [multiplier, 2 * multiplier, 3 * multiplier], + CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, + ].tolist() == [1, 2, 3] by_person = person.set_index("person_id")[CGT_RESIDENTIAL_GAINS_COLUMN] # Arm 1: gainer 0 residential; arm 2: gainer 1; arm 3: both. assert by_person.loc[[0 + multiplier, 1 + multiplier]].tolist() == [50_000.0, 0.0] @@ -252,6 +252,37 @@ def test_households_with_several_liable_gainers_become_product_arms() -> None: assert weights.loc[2] == pytest.approx(30.0) +def test_arms_take_their_households_ids_plus_j_times_the_declared_radix() -> None: + # Households come from the input by lineage (a marker column names each + # one), never from the arm's id: every id column of arm j sits at its + # household's plus j x 10**18, the copy-id table's top field + # (microcosm#931). + plain = _frame( + [50_000.0, 80_000.0, 90_000.0, 0.0, 1_000.0, 0.0], + weights=[10.0, 20.0, 30.0], + people_per_household=2, + ) + frame = uk_national_frame( + person=plain.table("person"), + benunit=plain.table("benunit"), + household=plain.table("household").assign(marker=[11, 12, 13]), + household_weights=plain.weights_for("household").values, + time_period="2024", + ) + result = split_cgt_residential_households( + frame, facts=_facts(frame), parameters=PARAMETERS + ) + + checked = assert_copy_ids_follow_lineage( + frame, + result.frame, + field=cgt_residential_split.CGT_RESIDENTIAL_COPY_ID_FIELD, + key_columns=("marker",), + ordinal_column=CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, + ) + assert checked == result.evidence()["arms_created"] == 4 + + def test_split_refuses_too_many_gainers_an_already_split_or_classified_frame() -> None: gains = [10_000.0] * (CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS + 1) frame = _frame(gains, people_per_household=len(gains)) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_structure.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_structure.py index d59d7b5d5a..b6d6cf318e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_structure.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_structure.py @@ -35,9 +35,13 @@ pair_clone_households, reporter_composition_quantiles, ) +from microcosm.build.uk_runtime.copy_id_table import radix from microcosm.build.uk_runtime.national_frame import uk_national_frame -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.frame import MassChangeRecord, WeightKind +from test_support.microcosm_build.uk_cgt_support import support_frame +from test_support.microcosm_build.uk_clone_first_frames import ( + assert_copy_ids_follow_lineage, +) def _distribution(*, negative: bool = False) -> dict[str, object]: @@ -156,10 +160,35 @@ def test_clone_splits_exact_mass_and_uses_oldest_adult_carriers() -> None: == 1 ) assert result.carrier_count == 2 - assert result.negative_prior_count == 1 + # The priors key on each twin's own person_id, minted at the copy-id + # table's cgt_incidence_clone radix (microcosm#931); the receipt counts + # the negative ones the draw kept. + assert result.negative_prior_count == int((gainers.capital_gains < 0.0).sum()) assert (gainers.capital_gains < 0.0).any() +def test_twins_take_their_originals_ids_plus_the_declared_radix() -> None: + # Originals come from the input by lineage (each household's council tax + # is its own here), never from the twin's id: every id column of a twin + # sits at its original's plus 10**17 (microcosm#931). + frame = support_frame( + weights=[10.0, 20.0, 30.0], + wealth=[0.0, 0.0, 0.0], + incomes=[20_000.0, 40_000.0, 60_000.0], + ages=[(45,), (50, 30, 8), (60, 58)], + benunits=[(0,), (0, 1, 1), (0, 1)], + ) + result = clone_cgt_incidence(frame, distribution=_distribution()) + + checked = assert_copy_ids_follow_lineage( + frame, + result.frame, + field="cgt_incidence_clone", + key_columns=("council_tax",), + ) + assert checked == 3 + + def test_prior_spline_keeps_negative_values_and_extrapolates_linearly() -> None: draws = _draw_banded_priors( np.zeros(4), @@ -326,13 +355,9 @@ def _anchor_input( full_pattern = list(pattern) mass_log: tuple[MassChangeRecord, ...] = () if support_copies: - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + # The support split's copies, as the stage mints them: copy k at k + # times the copy-id table's cgt_support_split radix (microcosm#931). + multiplier = radix("cgt_support_split") family = SUPPORT_COPY_COUNT + 1 old_total = float(source_weights.sum()) source_weights[0] = source_weights[0] / family @@ -417,8 +442,12 @@ def test_pairing_is_a_bijection_and_pairs_support_copies_with_their_clones() -> person, benunit, household ) - # The split copies' ids (1 + k x 100) push the clone multiplier to 1,000. - assert multiplier == 1_000 + # The twin's radix is declared (microcosm#931): the split copies' largest + # ordinal has one digit, so ten to the frame's digit count (10**14) would + # miss every original, and the pairing still finds each one. + assert multiplier == radix("cgt_incidence_clone") == 10**17 + originals = household.loc[~household[HOUSEHOLD_IS_CGT_CLONE], "household_id"] + assert 10 ** len(str(int(originals.max()))) == 10**14 assert len(clone_positions) == len(ANCHOR_PATTERN) + SUPPORT_COPY_COUNT ids = household["household_id"].to_numpy() assert (ids[clone_positions] - multiplier == ids[original_positions]).all() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 346594355a..c1e7ebb791 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -21,6 +21,7 @@ from microcosm.build.uk_runtime.cgt_support import ( CGT_SUPPORT_CLONE_SPLIT_FACTOR, CGT_SUPPORT_COPIES_COLUMN, + CGT_SUPPORT_COPY_BOUND_REFUSAL, CGT_SUPPORT_COPY_INDEX_COLUMN, CGT_SUPPORT_EXPECTED_MASS, CGT_SUPPORT_HEADROOM, @@ -42,12 +43,12 @@ select_cgt_support_households, split_cgt_support_households, ) +from microcosm.build.uk_runtime.copy_id_table import radix from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_INCOME_BAND_LOWER_BOUNDS, load_hmrc_cgt_joint_distribution, ) from microcosm.build.uk_runtime.national_frame import uk_national_frame -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.frame import WeightKind from test_support.microcosm_build.uk_cgt_support import ( BAND_INCOMES, @@ -58,10 +59,13 @@ support_stage, support_tables, ) +from test_support.microcosm_build.uk_clone_first_frames import ( + assert_copy_ids_follow_lineage, +) # --- the reference scenario ------------------------------------------------- # -# Seven one-person households (ids 1-7, multiplier 10). Band 0 (income +# Seven one-person households (ids 1-7; copies at k x 10**13). Band 0 (income # 20,000): ids 1-4 and the zero-weight id 7; band 37,700 (income 60,000): # ids 5-6. The synthetic joint publishes 60 top-band taxpayers in band 0 # (support mass 2 x 2 x 60 = 240, one suppressed cell) and 10 in band 37,700 @@ -94,6 +98,10 @@ SCENARIO_HEADROOM = 2.0 +#: The copy-id table's cgt_support_split radix (microcosm#931). +COPY_RADIX = radix("cgt_support_split") + + def _scenario_frame(**overrides): keywords = { "weights": SCENARIO_WEIGHTS, @@ -463,9 +471,9 @@ def test_split_conserves_mass_bitwise_and_corrects_only_an_unselected_incumbent( assert len(result.frame.mass_log) == len(frame.mass_log) + 1 weights = _household_weights(result.frame) # Roots and copies carry bitwise w / n: id 2 (130) in three, id 3 (121) - # in three, id 6 (61) in two, ids offset by k x 10. + # in three, id 6 (61) in two, ids offset by k x the declared radix. for household_id, weight, copies in ((2, 130.0, 3), (3, 121.0, 3), (6, 61.0, 2)): - family = [household_id + k * 10 for k in range(copies)] + family = [household_id + k * COPY_RADIX for k in range(copies)] assert weights.loc[family].tolist() == [weight / copies] * copies assert (weights.loc[family] <= CGT_SUPPORT_MAXIMUM_COPY_WEIGHT).all() # The uncorrected vector drifts by rounding (that is why this scenario was @@ -478,7 +486,7 @@ def test_split_conserves_mass_bitwise_and_corrects_only_an_unselected_incumbent( assert result.new_total == old -def test_split_offsets_every_entity_id_by_k_times_the_multiplier(): +def test_split_offsets_every_entity_id_by_k_times_the_declared_radix(): frame = _scenario_frame( ages=[(45,), (50, 30, 8), (45,), (45,), (45,), (60, 58), (45,)], benunits=[(0,), (0, 1, 1), (0,), (0,), (0,), (0, 1), (0,)], @@ -486,14 +494,10 @@ def test_split_offsets_every_entity_id_by_k_times_the_multiplier(): person = frame.table("person") benunit = frame.table("benunit") household = frame.table("household") - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) - assert multiplier == 100 + # The copy-id table's field, not a power of ten above the frame's largest + # id (microcosm#931). + multiplier = COPY_RADIX + assert multiplier == 10**13 result = split_cgt_support_households( frame, @@ -540,6 +544,46 @@ def test_split_offsets_every_entity_id_by_k_times_the_multiplier(): assert out_household["household_id"].dtype == np.dtype("int64") +def test_split_copies_take_their_roots_ids_plus_k_times_the_declared_radix(): + # Roots come from the input by lineage (each household's council tax is + # its own here, and copies carry it unchanged), never from the copy's id + # (microcosm#931). + frame = _scenario_frame( + ages=[(45,), (50, 30, 8), (45,), (45,), (45,), (60, 58), (45,)], + benunits=[(0,), (0, 1, 1), (0,), (0,), (0,), (0, 1), (0,)], + ) + result = split_cgt_support_households( + frame, + distribution=support_distribution(SCENARIO_TOP_CELLS), + parameters=PARAMETERS, + headroom=SCENARIO_HEADROOM, + ) + + checked = assert_copy_ids_follow_lineage( + frame, + result.frame, + field="cgt_support_split", + key_columns=("council_tax",), + ordinal_column=CGT_SUPPORT_COPY_INDEX_COLUMN, + ) + assert checked == result.copies_created == 5 + + +def test_split_refuses_a_household_beyond_the_copy_id_field_with_a_fixed_message(): + # Id 2 would need 10,001 copies of at most 60, one more than the + # cgt_support_split field's ordinals hold (microcosm#931). The refusal is + # a fixed sentence: never the household, its weight or its copy count. + weights = list(SCENARIO_WEIGHTS) + weights[1] = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * 10_000 + 1.0 + + with pytest.raises(ValueError) as caught: + _scenario_split(weights=weights) + + assert str(caught.value) == CGT_SUPPORT_COPY_BOUND_REFUSAL + assert "10001" not in str(caught.value) + assert "10,001" not in str(caught.value) + + def test_split_flags_only_copies_and_writes_the_family_copy_count(): result = _scenario_split() household = result.frame.table("household").set_index("household_id") @@ -547,11 +591,11 @@ def test_split_flags_only_copies_and_writes_the_family_copy_count(): assert household[HOUSEHOLD_IS_CGT_SUPPORT_COPY].dtype == np.dtype(bool) assert household[CGT_SUPPORT_COPIES_COLUMN].dtype == np.dtype("int64") assert household.index[household[HOUSEHOLD_IS_CGT_SUPPORT_COPY]].tolist() == [ - 12, - 13, - 16, - 22, - 23, + 2 + COPY_RADIX, + 3 + COPY_RADIX, + 6 + COPY_RADIX, + 2 + 2 * COPY_RADIX, + 3 + 2 * COPY_RADIX, ] assert household[CGT_SUPPORT_COPIES_COLUMN].to_dict() == { 1: 1, @@ -561,11 +605,11 @@ def test_split_flags_only_copies_and_writes_the_family_copy_count(): 5: 1, 6: 2, 7: 1, - 12: 3, - 13: 3, - 16: 2, - 22: 3, - 23: 3, + 2 + COPY_RADIX: 3, + 3 + COPY_RADIX: 3, + 6 + COPY_RADIX: 2, + 2 + 2 * COPY_RADIX: 3, + 3 + 2 * COPY_RADIX: 3, } assert result.households_before == 7 assert result.households_after == 12 @@ -601,7 +645,7 @@ def test_split_copies_every_column_and_membership_unchanged(): benunit.benunit_id.isin(root_people.person_benunit_id) ].reset_index(drop=True) for k in range(1, copies): - offset = k * 100 + offset = k * COPY_RADIX copy = household.loc[household.household_id == household_id + offset] # The explicit lineage: the root at 0, copy k at k. assert root[CGT_SUPPORT_COPY_INDEX_COLUMN].item() == 0 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py new file mode 100644 index 0000000000..24924f6e1a --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -0,0 +1,453 @@ +"""The UK copy-id table: fixed id fields for every copy layer (microcosm#931). + +Every id here is synthetic. The table declares each copy layer's decimal field +of the id; these tests hold its fields, its helpers and the code that mints +and decodes through it: the order of ids at K=1 (as the retired rule ordered +them), nesting across K on clone-first frames, the executor's parent recovery, +the static guard against the retired rule and the dense line's refusal. +""" + +from __future__ import annotations + +import ast +import inspect +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import microcosm.build as build +from microcosm.build.uk_runtime import copy_id_table, graph_kernels +from microcosm.build.uk_runtime.cgt_residential_split import ( + CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS, +) +from microcosm.build.uk_runtime.copy_id_table import ( + UK_COPY_ID_BASE_RADIX, + UK_COPY_ID_FIELDS, + UK_COPY_ID_LARGEST_ID, + CopyIdField, + mint, + ordinal, + ordinal_bound, + parent, + radix, + require_below, +) +from microcosm.build.uk_runtime.high_value_home_support import ( + HIGH_VALUE_SUPPORT_COPIES, +) +from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE +from microcosm.build.uk_runtime.rowwise_dataset import ( + expand_uk_geographic_pool, + refuse_dense_clone_id_overflow, +) +from microcosm.build.uk_runtime.spi_band_donors import ( + SPI_INCOME_BAND_DONOR_CLONE_INDEX, +) +from test_support.microcosm_build.uk_clone_first_frames import ( + COPY_PATH_COLUMNS, + GEOGRAPHIC_RADIX, + base_id_tables, + clone_first_tables, + copy_chain, + id_frame, +) + +_FIELD_NAMES = ( + "vintage", + "geographic_support", + "spi_support_channel", + "high_value_support", + "cgt_support_split", + "cgt_incidence_clone", + "cgt_residential_split", +) +#: The spine stage that mints each copy field (the band donors share the SPI +#: copies' field at ordinal 2). +_MINTING_STAGES = { + "spi_support_channel": "spi_support_channel", + "high_value_support": "high_value_home_support", + "cgt_support_split": "cgt_support_split", + "cgt_incidence_clone": "cgt_incidence_clone", + "cgt_residential_split": "cgt_residential_split", +} +_INT64_MAX = 2**63 - 1 + + +# --- the table ----------------------------------------------------------------- + + +def test_the_fields_bounds_and_radices_are_the_declared_ones() -> None: + assert tuple(field.name for field in UK_COPY_ID_FIELDS) == _FIELD_NAMES + assert {field.name: (field.bound, field.radix) for field in UK_COPY_ID_FIELDS} == { + "vintage": (10, 10**8), + "geographic_support": (100, 10**9), + "spi_support_channel": (3, 10**11), + "high_value_support": (6, 10**12), + "cgt_support_split": (10_000, 10**13), + "cgt_incidence_clone": (2, 10**17), + "cgt_residential_split": (8, 10**18), + } + assert UK_COPY_ID_BASE_RADIX == 10**8 + # The CGT field holds four decimal digits. + assert copy_id_table._field("cgt_support_split").width == 10**4 + + +def test_the_largest_mintable_id_fits_int64() -> None: + expected = ( + sum((field.bound - 1) * field.radix for field in UK_COPY_ID_FIELDS) + 10**8 - 1 + ) + assert UK_COPY_ID_LARGEST_ID == expected + assert 7.1e18 < UK_COPY_ID_LARGEST_ID < 7.2e18 < _INT64_MAX + # Every field's largest ordinal stays inside it: the next field starts + # where this one's width ends. + for lower, upper in zip(UK_COPY_ID_FIELDS, UK_COPY_ID_FIELDS[1:], strict=False): + assert lower.radix * lower.width == upper.radix + + +def test_the_copy_fields_are_the_lineage_branches_in_stage_order() -> None: + lineage = {branch.name for branch in UK_COPY_LINEAGE.branches} + assert set(_FIELD_NAMES) - {"vintage"} == lineage + # Application order, not the lineage's branch order: each field's stage + # runs after every lower field's, so a copy's parent sits below its radix. + payload = json.loads( + (Path(build.__file__).parent / "uk" / "source_stages.json").read_text() + ) + position = {stage["stage"]: i for i, stage in enumerate(payload["stages"])} + order = [ + position[_MINTING_STAGES[name]] + for name in _FIELD_NAMES + if name in _MINTING_STAGES + ] + assert order == sorted(order) + assert position["spi_income_band_donors"] < position["high_value_home_support"] + + +def test_every_bound_covers_its_stage_constant() -> None: + assert HIGH_VALUE_SUPPORT_COPIES <= ordinal_bound("high_value_support") + assert 2**CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS <= ordinal_bound( + "cgt_residential_split" + ) + assert SPI_INCOME_BAND_DONOR_CLONE_INDEX < ordinal_bound("spi_support_channel") + assert ordinal_bound("cgt_support_split") == 10_000 + assert ordinal_bound("geographic_support") == 100 + + +def test_a_malformed_table_is_refused() -> None: + fields = list(UK_COPY_ID_FIELDS) + # Ten arms over ten incidence ordinals would mint above int64. + wide = fields[:5] + [ + CopyIdField("cgt_incidence_clone", 10, 10**17), + CopyIdField("cgt_residential_split", 10, 10**18), + ] + with pytest.raises(ValueError, match="int64 cannot hold"): + copy_id_table._validate_fields(wide, 10**8) + gap = [*fields[:2], CopyIdField("spi_support_channel", 3, 10**12), *fields[3:]] + with pytest.raises(ValueError, match="not contiguous"): + copy_id_table._validate_fields(gap, 10**8) + with pytest.raises(ValueError, match="each once"): + copy_id_table._validate_fields([fields[0], fields[0]], 10**8) + with pytest.raises(ValueError, match="at least 2"): + copy_id_table._validate_fields([CopyIdField("vintage", 1, 10**8)], 10**8) + with pytest.raises(ValueError, match="No UK copy-id field"): + radix("clone_index") + + +# --- the helpers ---------------------------------------------------------------- + + +def test_mint_ordinal_and_parent_round_trip_in_every_field() -> None: + rng = np.random.default_rng(931) + base = rng.integers(1, 10**8, size=200, dtype=np.int64) + ids = base.copy() + ordinals = {} + for field in UK_COPY_ID_FIELDS: + drawn = rng.integers(1, field.bound, size=len(ids), dtype=np.int64) + ids = mint(ids, field.name, drawn) + ordinals[field.name] = drawn + for field in reversed(UK_COPY_ID_FIELDS): + np.testing.assert_array_equal(ordinal(ids, field.name), ordinals[field.name]) + ids = parent(ids, field.name) + np.testing.assert_array_equal(ordinal(ids, field.name), 0) + np.testing.assert_array_equal(ids, base) + assert mint(np.zeros(0, dtype=np.int64), "cgt_support_split", 1).size == 0 + + +def test_mint_refuses_every_malformed_copy_without_naming_an_id() -> None: + ids = np.asarray([5, 7_000_000_042], dtype=np.int64) + # An ordinal outside the field is refused in one fixed sentence. + outside = ( + "A 'high_value_support' copy ordinal falls outside [1, 6), the field's " + "declared bound (microcosm#931)." + ) + for ordinals in (0, [1, 6], [6, 7]): + with pytest.raises(ValueError) as caught: + mint(ids, "high_value_support", ordinals) + assert str(caught.value) == outside + with pytest.raises(ValueError, match=r"^1 id\(s\) already carry"): + mint(ids + np.asarray([0, 10**12]), "high_value_support", 1) + # Empty CGT digits under higher ones near the top of int64. + near_top = np.asarray([9_200_000_000_000_000_000], dtype=np.int64) + with pytest.raises(ValueError, match="overflow int64") as caught: + mint(near_top, "cgt_support_split", 2_400) + assert str(int(near_top[0])) not in str(caught.value) + with pytest.raises(ValueError, match="one integer or one per id"): + mint(ids, "high_value_support", [1, 2, 3]) + with pytest.raises(TypeError, match="must be integers"): + mint(ids.astype(float), "high_value_support", 1) + with pytest.raises(TypeError, match="must be integers"): + mint(ids, "high_value_support", True) + with pytest.raises(ValueError, match="negative"): + mint(np.asarray([-3]), "high_value_support", 1) + + +def test_require_below_refuses_ids_at_the_radix_printing_only_counts() -> None: + require_below(np.asarray([1, 10**12 - 1]), "high_value_support") + secret = 10**12 + 4_321 + with pytest.raises(ValueError, match=r"^2 of the incoming ids reach") as caught: + require_below( + pd.Series([3, 10**12, secret]), "high_value_support", what="incoming ids" + ) + assert str(secret) not in str(caught.value) + # Object columns of Python ints are read exactly, above 2**53 too. + require_below(pd.Series([2**53 + 1], dtype=object), "cgt_residential_split") + + +def test_the_frs_spine_refuses_raw_ids_that_reach_the_first_copy_field() -> None: + from microcosm.build.uk_runtime import frs_spine + + tables = base_id_tables(3) + frs_spine._require_raw_ids_below_copy_fields(id_frame(tables)) + # A sernum of 100,000 gives person ids from 10**8, the vintage field. + person = tables.person.copy() + person.loc[person.index[-1], "person_id"] = 10**8 + 1 + with pytest.raises(ValueError, match=r"^1 of the raw FRS person_id values reach"): + frs_spine._require_raw_ids_below_copy_fields( + id_frame(type(tables)(person, tables.benunit, tables.household)) + ) + + +# --- order and nesting on synthetic chains --------------------------------------- + + +def test_k1_ids_sort_as_the_retired_rule_sorted_them_and_base_ids_keep() -> None: + base = base_id_tables() + table = copy_chain(base, rule="table") + retired = copy_chain(base, rule="retired") + for entity in ("household", "benunit", "person"): + np.testing.assert_array_equal( + np.argsort(table.ids(entity), kind="stable"), + np.argsort(retired.ids(entity), kind="stable"), + ) + path = table.household[list(COPY_PATH_COLUMNS)].to_numpy() + originals = (path == 0).all(axis=1) + np.testing.assert_array_equal( + table.household["household_id"].to_numpy()[originals], + base.household["household_id"].to_numpy(), + ) + np.testing.assert_array_equal( + table.household["household_id"].to_numpy()[originals], + retired.household["household_id"].to_numpy()[originals], + ) + # Every layer copied something, and the CGT ordinals reach two digits. + assert table.household[list(COPY_PATH_COLUMNS)].max().to_dict() == { + "spi": 2, + "hv": 5, + "cgt": 11, + "twin": 1, + "arm": 7, + } + assert int(table.household["household_id"].max()) > 2**53 + + +def _clone_ids(tables, entity: str, clone: int) -> np.ndarray: + household_clone = tables.household.set_index("household_id")[ + "household_clone_index" + ] + if entity == "household": + mask = tables.household["household_clone_index"].to_numpy() == clone + return np.sort(tables.ids("household")[mask]) + person_clone = tables.person["person_household_id"].map(household_clone) + if entity == "person": + return np.sort(tables.ids("person")[person_clone.to_numpy() == clone]) + units = set( + tables.person.loc[person_clone.to_numpy() == clone, "person_benunit_id"] + ) + return np.sort( + tables.ids("benunit")[tables.benunit["benunit_id"].isin(units).to_numpy()] + ) + + +def test_clone_zero_and_its_copies_keep_the_k1_ids_on_clone_first_frames() -> None: + base = base_id_tables() + k1 = copy_chain(clone_first_tables(base, 1), rule="table") + k3 = copy_chain(clone_first_tables(base, 3), rule="table") + for entity in ("household", "benunit", "person"): + k1_ids = _clone_ids(k1, entity, 0) + assert len(k1_ids) == len(k1.ids(entity)) + np.testing.assert_array_equal(_clone_ids(k3, entity, 0), k1_ids) + for clone in (1, 2): + ids = _clone_ids(k3, entity, clone) + np.testing.assert_array_equal( + ids, np.sort(k1_ids + clone * GEOGRAPHIC_RADIX) + ) + assert not set(ids.tolist()) & set(k1_ids.tolist()) + # Ten to the frame's digit count would give clone 0 other ids at K=3. + r1 = copy_chain(clone_first_tables(base, 1), rule="retired") + r3 = copy_chain(clone_first_tables(base, 3), rule="retired") + assert not np.array_equal( + _clone_ids(r3, "household", 0), _clone_ids(r1, "household", 0) + ) + + +# --- the executor --------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("stage", "field", "layers"), + [ + ("high_value_home_support", "high_value_support", 2), + ("cgt_support_split", "cgt_support_split", 3), + ("cgt_incidence_clone", "cgt_incidence_clone", 4), + ("cgt_residential_split", "cgt_residential_split", 5), + ], +) +def test_the_executor_recovers_each_copy_parent_at_the_declared_radix( + stage, field, layers +) -> None: + base = base_id_tables() + before = id_frame(copy_chain(base, layers=layers)) + after = id_frame(copy_chain(base, layers=layers + 1)) + offset = graph_kernels._declared_copy_radix(stage, before) + assert offset == radix(field) + for entity in before.entities: + lineage = graph_kernels._source_lineage(before, after, entity, id_offset=offset) + targets = lineage.index.to_numpy(dtype=np.int64) + assert len(targets) > 0 + assert lineage.dtype == np.dtype("int64") + np.testing.assert_array_equal(lineage.to_numpy(), parent(targets, field)) + assert set(lineage.tolist()) <= set( + before.table(entity)[before.schema.entity_id_column(entity)].tolist() + ) + + +def test_the_executor_stays_exact_above_2_53_on_all_integer_tables() -> None: + # The person table carries integer columns only, which a row-by-row walk + # would have read back as float64. + base = base_id_tables() + before = id_frame(copy_chain(base, layers=5)) + after = id_frame(copy_chain(base)) + lineage = graph_kernels._source_lineage( + before, after, "person", id_offset=radix("cgt_residential_split") + ) + targets = lineage.index.to_numpy(dtype=np.int64) + assert int(targets.min()) > 2**53 + np.testing.assert_array_equal( + lineage.to_numpy(), targets % radix("cgt_residential_split") + ) + + +def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: + base = base_id_tables() + before = id_frame(copy_chain(base, layers=4)) + assert graph_kernels._declared_copy_radix("frs_council_tax", before) is None + # A frame that already carries twins reaches the twin's radix. + twinned = copy_chain(base, layers=5) + with pytest.raises( + ValueError, match=r"^\d+ of the UK stage 'cgt_incidence_clone' incumbent" + ) as caught: + graph_kernels._declared_copy_radix("cgt_incidence_clone", id_frame(twinned)) + largest = int(twinned.household["household_id"].max()) + assert str(largest) not in str(caught.value) + + +# --- no UK spine module keeps the retired rule ------------------------------------ + + +@pytest.mark.parametrize( + "module", + [ + "spi_support", + "spi_band_donors", + "high_value_home_support", + "cgt_support", + "cgt_structure", + "cgt_residential_split", + "graph_kernels", + ], +) +def test_no_uk_spine_stage_module_imports_the_retired_id_rule(module) -> None: + import importlib + + source = Path( + inspect.getfile(importlib.import_module(f"microcosm.build.uk_runtime.{module}")) + ).read_text(encoding="utf-8") + names = set() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.ImportFrom): + names.update(alias.name for alias in node.names) + elif isinstance(node, ast.Attribute): + names.add(node.attr) + elif isinstance(node, ast.Name): + names.add(node.id) + assert "id_multiplier_for_values" not in names + + +# --- the dense line --------------------------------------------------------------- + + +def _pool_inputs(tables): + # A spine carries no clone index and no test path columns. + household = tables.household.drop( + columns=["household_clone_index", *COPY_PATH_COLUMNS] + ) + household["household_weight"] = 1.0 + household["region"] = "LONDON" + return { + "person": tables.person, + "benunit": tables.benunit, + "household": household, + "source_year": 2024, + "time_period": "2024", + } + + +def test_dense_clones_of_a_table_minted_spine_are_refused_in_one_message() -> None: + spine = copy_chain(base_id_tables()) + with pytest.raises(ValueError) as caught: + expand_uk_geographic_pool(n_clones=2, **_pool_inputs(spine)) + message = str(caught.value) + assert "microcosm#931" in message + assert "clone-first spine" in message + assert "n_clones=2" in message + assert str(int(spine.household["household_id"].max())) not in message + with pytest.raises(ValueError, match="clone-first spine"): + refuse_dense_clone_id_overflow( + n_clones=15, id_multiplier=10**18, ids=(spine.ids("household"),) + ) + + +def test_one_dense_clone_of_a_table_minted_spine_still_expands() -> None: + spine = copy_chain(base_id_tables()) + pool = expand_uk_geographic_pool(n_clones=1, **_pool_inputs(spine)) + np.testing.assert_array_equal( + pool.frame.table("household")["household_id"].to_numpy(), + spine.ids("household"), + ) + np.testing.assert_array_equal( + pool.frame.table("person")["person_id"].to_numpy(), spine.ids("person") + ) + + +def test_a_spine_without_copy_rows_still_expands_into_dense_clones() -> None: + base = base_id_tables() + pool = expand_uk_geographic_pool(n_clones=3, **_pool_inputs(base)) + # Today's dense rule: clones above ten to the spine's digit count. + multiplier = 10 ** len(str(int(base.person["person_id"].max()))) + assert pool.id_multiplier == multiplier + np.testing.assert_array_equal( + pool.frame.table("household")["household_id"].to_numpy(), + np.concatenate([base.ids("household") + k * multiplier for k in range(3)]), + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 2e76bfd799..75052ee434 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -307,6 +307,79 @@ def changed_helper_bytes(path: Path, helper=helper_path, read=original): assert registry.implementation_hash(control) == before[control] +#: The two SPI support stages mint their copies' ids through the support +#: channel's clone helper (``spi_support``), which reads the copy-id table; +#: every other minting stage's module reads the table itself (microcosm#931). +_SPI_COPY_ID_STAGES = frozenset({"spi_support_channel", "spi_income_band_donors"}) + + +def test_every_stage_that_mints_or_bounds_copy_ids_binds_the_copy_id_table() -> None: + """``source_hash`` does not follow imports, so every stage whose module + reads the copy-id table, and the two SPI support stages, whose minting + runs in spi_support, bind it; no other stage does (microcosm#931).""" + from microcosm.build.uk_runtime import copy_id_table, graph_kernels, spi_support + + assert "copy_id_table" in _imported_module_names(spi_support) + reading = { + stage + for stage in graph_kernels._STAGE_MODULES + if "copy_id_table" in _imported_module_names(graph_kernels._stage_module(stage)) + } + assert reading == { + "frs_spine", + "high_value_home_support", + "cgt_support_split", + "cgt_incidence_clone", + "cgt_incidence_anchor", + "cgt_residential_split", + } + binding = { + stage + for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() + if copy_id_table in helpers + } + assert binding == reading | _SPI_COPY_ID_STAGES + for stage in _SPI_COPY_ID_STAGES: + assert spi_support in graph_kernels._STAGE_HELPER_MODULES[stage] + + +def test_copy_id_table_changes_invalidate_every_stage_that_reads_it(monkeypatch): + from microcosm.build.uk_runtime import copy_id_table, graph_kernels + + graph = uk_spine_graph() + registry = uk_registry(graph=graph) + stages = sorted( + stage + for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() + if copy_id_table in helpers + ) + # The root stage's node is the CREATE kernel. + affected = [ + graph.node("create_uk_frs" if stage == "frs_spine" else stage).kernel + for stage in stages + ] + controls = [ + graph.node(stage).kernel + for stage in ("frs_council_tax", "hmrc_spi_income_spine", "isa_subscriptions") + ] + before = {ref: registry.implementation_hash(ref) for ref in affected + controls} + helper_path = Path(copy_id_table.__file__).resolve() + original = Path.read_bytes + + def changed_helper_bytes(path: Path) -> bytes: + content = original(path) + if path.resolve() == helper_path: + return content + b"\n# A helper-only source change.\n" + return content + + monkeypatch.setattr(Path, "read_bytes", changed_helper_bytes) + assert len(affected) == 8 + for ref in affected: + assert registry.implementation_hash(ref) != before[ref], ref + for ref in controls: + assert registry.implementation_hash(ref) == before[ref], ref + + def test_isa_statistics_source_changes_invalidate_the_stages_that_read_them( monkeypatch, ): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py index d481456d4e..25429cdb82 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py @@ -17,6 +17,9 @@ from microcosm.build.uk_runtime.property_localisation import SUPPORT_STRATUM_LOWER from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate from microcosm.frame import WeightKind +from test_support.microcosm_build.uk_clone_first_frames import ( + assert_copy_ids_follow_lineage, +) def _frame(n: int = 40): @@ -118,6 +121,40 @@ def test_split_copies_whole_households_and_conserves_mass() -> None: hv.split_high_value_support_households(out) +def test_copies_take_their_roots_ids_plus_k_times_the_declared_radix() -> None: + # Roots come from the input by lineage, never from the copy's id: every + # id column of copy k sits at its root's plus k x 10**12 (microcosm#931). + frame = _frame() + result = hv.split_high_value_support_households(frame, stratum_lower=0.9) + + checked = assert_copy_ids_follow_lineage( + frame, + result.frame, + field=hv.HIGH_VALUE_SUPPORT_COPY_ID_FIELD, + key_columns=("source_household_id",), + ordinal_column=HIGH_VALUE_SUPPORT_INDEX_COLUMN, + ) + assert checked == result.copies_created == 4 * (hv.HIGH_VALUE_SUPPORT_COPIES - 1) + + +def test_the_split_refuses_incoming_ids_at_its_copy_id_radix() -> None: + frame = _frame() + household = frame.table("household").copy() + person = frame.table("person").copy() + household.loc[household.index[-1], "household_id"] = 10**12 + 40 + person.loc[person.index[-1], "person_household_id"] = 10**12 + 40 + shifted = uk_national_frame( + person=person, + benunit=frame.table("benunit"), + household=household, + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=frame.weights_for("household").values, + ) + with pytest.raises(ValueError, match=r"^1 of the incoming person_household_id"): + hv.split_high_value_support_households(shifted) + + def test_evidence_matches_the_gate_and_the_committed_declaration() -> None: frame = _frame() result = hv.split_high_value_support_households(frame) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index 452f40f138..822af31afd 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -42,13 +42,18 @@ prepare_spi_donor_table, ) from microcosm.build.uk_runtime.spi_support import ( + BASE_FRS_SUPPORT_CHANNEL, HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, SPI_PROPERTY_FINANCE_COSTS_COLUMN, SPI_SYNTHETIC_SUPPORT_CHANNEL, build_uk_spi_support_channel, support_channel_column, + support_clone_index_column, ) from microcosm.frame import WeightKind +from test_support.microcosm_build.uk_clone_first_frames import ( + assert_copy_ids_follow_lineage, +) from test_support.paths import paths_for _IDENTITY_TOOL_PATH = ( @@ -368,6 +373,41 @@ def test_stack_reserves_band_rows_and_conserves_every_region_mass() -> None: assert (weights.to_numpy()[:incumbents][~positive] == 0.0).all() +def test_spi_copies_and_donors_take_their_base_roots_ids_plus_the_radix() -> None: + # Every SPI-channel household copies a base household, which its + # household_source_id names: the SPI copy at 1 and the income band donor at + # 2 times the spi_support_channel radix, on every id column (microcosm#931). + frame = _support_frame(200) + result = _stack_small(frame) + channel = support_channel_column("household") + household = frame.table("household") + base = household.loc[household[channel] == BASE_FRS_SUPPORT_CHANNEL] + person = frame.table("person") + before = { + "household": base, + "person": person.loc[person["person_household_id"].isin(base["household_id"])], + "benunit": frame.table("benunit"), + } + + checked = assert_copy_ids_follow_lineage( + before, + result.frame, + field="spi_support_channel", + key_columns=("household_source_id",), + ordinal_column=support_clone_index_column("household"), + ) + donors = result.frame.table("household")[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR] + after_channel = result.frame.table("household")[channel] + assert checked == int((after_channel == SPI_SYNTHETIC_SUPPORT_CHANNEL).sum()) + assert int(donors.sum()) > 0 + assert ( + result.frame.table("household") + .loc[donors, support_clone_index_column("household")] + .eq(SPI_INCOME_BAND_DONOR_CLONE_INDEX) + .all() + ) + + def test_stack_scales_seats_then_mass_on_a_frame_too_small_for_the_plan() -> None: # Given the published masses on a 60-household frame, the plan's 1,920 # donors do not fit: the seats shrink to the declared share of the frame. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py index 5672e79fe8..fb7348f38d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py @@ -25,6 +25,9 @@ ) from microcosm.build.uk_runtime.terminal_gates import UKZeroWeightStratumDeclaration from microcosm.frame import MassChangeRecord, WeightKind +from test_support.microcosm_build.uk_clone_first_frames import ( + assert_copy_ids_follow_lineage, +) def household_frame() -> pd.DataFrame: @@ -66,15 +69,17 @@ def test_spi_support_creates_zero_weight_copy_with_lineage_and_remapped_ids() -> source_year=2023, ) - assert result.id_multiplier == 10_000 + # The SPI copies take the copy-id table's spi_support_channel radix, not + # a power of ten above the frame's largest id (microcosm#931). + assert result.id_multiplier == 10**11 assert result.n_spi_households == 3 assert result.household["household_id"].tolist() == [ 1, 2, 3, - 10001, - 10002, - 10003, + 10**11 + 1, + 10**11 + 2, + 10**11 + 3, ] assert result.household["household_weight"].tolist() == [ 10.0, @@ -132,32 +137,88 @@ def test_spi_support_creates_zero_weight_copy_with_lineage_and_remapped_ids() -> 2001, 2002, 3001, - 11001, - 12001, - 12002, - 13001, + 10**11 + 1001, + 10**11 + 2001, + 10**11 + 2002, + 10**11 + 3001, ] assert result.person["person_household_id"].tolist() == [ 1, 2, 2, 3, - 10001, - 10002, - 10002, - 10003, + 10**11 + 1, + 10**11 + 2, + 10**11 + 2, + 10**11 + 3, ] assert result.person["person_benunit_id"].tolist() == [ 101, 201, 201, 301, - 10101, - 10201, - 10201, - 10301, + 10**11 + 101, + 10**11 + 201, + 10**11 + 201, + 10**11 + 301, ] - assert result.benunit["benunit_id"].tolist() == [101, 201, 301, 10101, 10201, 10301] + assert result.benunit["benunit_id"].tolist() == [ + 101, + 201, + 301, + 10**11 + 101, + 10**11 + 201, + 10**11 + 301, + ] + + +def test_spi_copies_take_their_roots_ids_plus_the_declared_radix() -> None: + # Roots come from the input by lineage (region names one household here), + # never from the copy's id (microcosm#931). + before = { + "person": person_frame(), + "benunit": benunit_frame(), + "household": household_frame(), + } + result = create_uk_spi_support_tables( + **before, spi_household_count=None, source_year=2023 + ) + after = { + "person": result.person, + "benunit": result.benunit, + "household": result.household, + } + + assert ( + assert_copy_ids_follow_lineage( + before, + after, + field="spi_support_channel", + key_columns=("region",), + ordinal_column=support_clone_index_column("household"), + ) + == 3 + ) + + +def test_spi_support_refuses_incoming_ids_that_reach_its_copy_id_field() -> None: + household = household_frame() + household.loc[2, "household_id"] = 10**11 + 3 + person = person_frame() + person.loc[3, "person_household_id"] = 10**11 + 3 + + with pytest.raises( + ValueError, match=r"^1 of the incoming household_id values reach" + ) as caught: + create_uk_spi_support_tables( + person=person, + benunit=benunit_frame(), + household=household, + spi_household_count=None, + source_year=2023, + ) + # A count, never the id. + assert str(10**11 + 3) not in str(caught.value) def test_spi_support_can_subsample_without_rescaling_or_reordering_base() -> None: diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json index 7f3e282ca1..be9eb32626 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json @@ -2,70 +2,70 @@ "resource": "synthetic-cgt-asset-type.json", "resource_sha256": "synthetic", "source_commit": "synthetic", - "table4_all_gains": 6147151.073180825, + "table4_all_gains": 7278689.42603867, "table4_all_tax": 0.0, - "table4_all_taxpayers": 15.617550412698908, + "table4_all_taxpayers": 17.24436988246394, "table4_bands": [ { - "gains": 3996.90576475932, + "gains": 7949.502620452311, "lower_bound": 0, "tax": null, - "taxpayers": 1.02613414910079, + "taxpayers": 1.9673816546685436, "upper_bound": 10000 }, { - "gains": 41562.81142286275, + "gains": 26946.918634102105, "lower_bound": 10000, "tax": null, - "taxpayers": 2.4014744153664607, + "taxpayers": 1.7272922430098223, "upper_bound": 25000 }, { - "gains": 96119.20133089423, + "gains": 51102.16729150777, "lower_bound": 25000, "tax": null, - "taxpayers": 2.6899006620693813, + "taxpayers": 1.3784835361616843, "upper_bound": 50000 }, { - "gains": 103973.22258628672, + "gains": 144887.97165642734, "lower_bound": 50000, "tax": null, - "taxpayers": 1.4354631379984855, + "taxpayers": 2.0034797890802527, "upper_bound": 100000 }, { - "gains": 159748.21024802336, + "gains": 289261.98743372096, "lower_bound": 100000, "tax": null, - "taxpayers": 0.9933755404532624, + "taxpayers": 1.7329974891738054, "upper_bound": 250000 }, { - "gains": 417023.37007052475, + "gains": 554667.0416390727, "lower_bound": 250000, "tax": null, - "taxpayers": 1.2096898937014449, + "taxpayers": 1.6027496904998344, "upper_bound": 500000 }, { - "gains": 1233932.188532695, + "gains": 1628625.8536635868, "lower_bound": 500000, "tax": null, - "taxpayers": 1.7707174507843035, + "taxpayers": 2.2567374967701963, "upper_bound": 1000000 }, { - "gains": 4090795.1632247795, + "gains": 4575247.983099801, "lower_bound": 1000000, "tax": null, - "taxpayers": 4.090795163224779, + "taxpayers": 4.575247983099801, "upper_bound": null } ], - "table4_individuals_gains": 6147151.073180825, + "table4_individuals_gains": 7278689.42603867, "table4_individuals_tax": 0.0, - "table4_individuals_taxpayers": 15.617550412698908, + "table4_individuals_taxpayers": 17.24436988246394, "table4_trusts_gains": 0.0, "table4_trusts_tax": 0.0, "table7_total_disposals": 2743095.2380952383, @@ -114,12 +114,12 @@ "proceeds": 9000000000.0 } ], - "table8a_disposals_total": 29.45023792108938, - "table8a_gains_total": 34966073.73417176, - "table8a_tax_total": 6993214.746834353, - "table8a_taxpayers_total": 26.772943564626704, - "table8b_all_gains": 34966073.73417176, - "table8b_all_taxpayers": 26.772943564626704, - "table8b_individuals_gains": 34966073.73417176, - "table8b_individuals_taxpayers": 26.772943564626704 + "table8a_disposals_total": 32.51795463550343, + "table8a_gains_total": 43141212.12856026, + "table8a_tax_total": 8628242.425712053, + "table8a_taxpayers_total": 29.56177694136675, + "table8b_all_gains": 43141212.12856026, + "table8b_all_taxpayers": 29.56177694136675, + "table8b_individuals_gains": 43141212.12856026, + "table8b_individuals_taxpayers": 29.56177694136675 } diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 9d079c4d6c..3e15f4398b 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -325,9 +325,9 @@ "operations": [ { "clone_flag_column": "household_is_capital_gains_clone", - "id_remapping": "id_multiplier_for_values", + "id_remapping": "copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17)", "kind": "pair_clone_households_to_originals", - "pairing": "clone household_id minus the clone stage's id multiplier, recomputed from the pre-clone rows (clone flag false; the support split's copies are pre-clone rows with their own clones)", + "pairing": "clone household_id minus the cgt_incidence_clone field's declared radix (the support split's copies are pre-clone rows with their own clones)", "requirement": "bijection between clone and original households; an unpaired clone or original fails the build" }, { @@ -406,7 +406,7 @@ "declared_factor": 1, "entity": "household", "flag_column": "household_is_capital_gains_clone", - "id_remapping": "id_multiplier_for_values", + "id_remapping": "copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17)", "kind": "clone_records", "mass_split": 0.5, "original_flag": false, @@ -483,7 +483,7 @@ "cgt_residential_probability", "capital_gains_residential_property" ], - "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the id multiplier, the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time.", + "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time.", "operations": [ { "artifact_role": "cgt_asset_type_facts", @@ -509,7 +509,7 @@ "flag_column": "household_is_cgt_residential_clone", "gainer_order": "person_id ascending within the household", "gains_target": "Table 8a 2024-25 total residential property gains times the Table 8b 2024-25 individuals/all gains share on the UK Property service", - "id_remapping": "id_multiplier_for_values; arm j takes every entity id + j x multiplier, bit m of j marking the household's m-th liable gainer residential", + "id_remapping": "copy-id table field cgt_residential_split, the top field; arm j takes every entity id + j x the field's radix (10**18), bit m of j marking the household's m-th liable gainer residential", "index_column": "cgt_residential_clone_index", "kind": "split_liable_gainers_by_residential_probability", "maximum_liable_gainers_per_household": 3, @@ -578,7 +578,7 @@ "expected_support_mass": 145000, "flag_column": "household_is_cgt_support_copy", "headroom": 1.25, - "id_remapping": "id_multiplier_for_values; copy k takes every entity id + k x multiplier", + "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "income_band_lower_bounds": [ 0, 37700, @@ -3090,7 +3090,7 @@ "copy_rule": "every selected household becomes copies rows at weight / copies", "declared_factor": 1, "flag_column": "household_is_high_value_support_copy", - "id_remapping": "id_multiplier_for_values; copy k takes every entity id + k x multiplier", + "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", "kind": "split_top_value_owner_households", "reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "stratum_lower": 0.95, @@ -7419,6 +7419,7 @@ "funding_strata": [ "region" ], + "id_remapping": "copy-id table field spi_support_channel; a donor takes every entity id + 2 x the field's radix (10**11), and its source is read from its source-id columns, never from the id", "kind": "stack_income_band_donor_households", "mass_scale": "min(1, min over funding strata of maximum_reallocation_share x stratum mass / seated donor mass); every band's weight scaled alike", "maximum_donor_household_share": 0.15, @@ -7484,6 +7485,7 @@ "count": 10000, "draw": "uniform_without_replacement", "flag_column": "household_is_spi_synthetic", + "id_remapping": "copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11)", "kind": "stack_zero_weight_donors", "seed": 42 }, diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_purchase_targets.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_purchase_targets.json index 94bcaee908..33c37856ec 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_purchase_targets.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_purchase_targets.json @@ -26,8 +26,8 @@ ], "minimum_price": 1000.0, "property_type": "additional_residential", - "transactions": 17.498825077930587, - "value": 110324.32813304412 + "transactions": 16.930405602541164, + "value": 132441.7306215811 }, "derivation": {}, "non_residential": { @@ -57,7 +57,7 @@ ], "minimum_price": 1000.0, "property_type": "non_residential", - "transactions": 7.755511160450606, - "value": 73068.4056805093 + "transactions": 8.346596362360707, + "value": 125282.69039109821 } } diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_rental_facts.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_rental_facts.json index dc9150e0e8..700e6fda1a 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_rental_facts.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_rental_facts.json @@ -9,18 +9,18 @@ "services_including_wages": 67.81449346944132 }, "expenses": { - "all": 35559.82465181411, - "individual": 35204.226405295965, - "partnership": 355.59824651814415 + "all": 32757.922728603626, + "individual": 32430.343501317588, + "partnership": 327.5792272860381 }, "expenses_by_type": { - "legal_management_and_professional_fees": 4256.9459151524225, - "non_residential_finance_costs": 1360.994727680943, - "other_allowable_expenses": 4686.7337238937735, - "rent_rates_and_insurance": 3898.789407867964, - "repairs_and_maintenance": 6559.3806048382285, - "residential_finance_costs": 13118.761209676457, - "services_including_wages": 1678.2190627043205 + "legal_management_and_professional_fees": 3921.52398707888, + "non_residential_finance_costs": 1253.756467022815, + "other_allowable_expenses": 4317.44708192819, + "rent_rates_and_insurance": 3591.588074704455, + "repairs_and_maintenance": 6042.540566628755, + "residential_finance_costs": 12085.08113325751, + "services_including_wages": 1545.98541798302 }, "landlords": { "all": 135.62898693888263, @@ -28,9 +28,9 @@ "partnership": 1.3562898693888314 }, "receipts": { - "all": 56895.71944290258, - "individual": 56326.76224847355, - "partnership": 568.9571944290292 + "all": 52412.6763657658, + "individual": 51888.54960210814, + "partnership": 524.1267636576595 }, "receipts_bands": [ { diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index b8bc94fe0b..191c66627e 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"2c682605139c0a8f96e8ba45a2c03f506d3f758ef437841066a8fdb8b74269b6","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"b67ec0b7c62b9c759c0615092cf111c3d671608648315d9657595df1e0ee80b1","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"174f4b38228d98107bee1639d1a297a2748bb618e87c7bd0a6c869c6ab06f8fa","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"be1a93225103ce878603d9b72e44cb7ef871d2443b95807de10caab4d36dce64","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"29198953c5522c9b996b981478909916870977a79e9f2feb2db7e9e72ef15de8","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"ed54c55428e56d7597c57c8459b014aa43b2ff9495b903f0b789bd5e4264ca9a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"d27f0a6bc984fbbe83d50d60a9037ee6a693062306039abfa2dc5418b971b817","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"48b7648953aabc721c6672038770b18a1ff7c308d71bd982770b8a3fe259280f","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"85696cd009ce349a132f753c55cb2168ed2f548de9f628ce74a71e99a0e9e6ba","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"5dcae0fb0c93d443b727ea728c5e61e11cdc4a1ad6f01f728b58eb8419760629","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} diff --git a/test_support/microcosm_build/uk_clone_first_frames.py b/test_support/microcosm_build/uk_clone_first_frames.py new file mode 100644 index 0000000000..101f1ddab3 --- /dev/null +++ b/test_support/microcosm_build/uk_clone_first_frames.py @@ -0,0 +1,333 @@ +"""Synthetic UK id frames for the copy-id tests (microcosm#931). + +The clone-first spine (#931 group (b)) will mint geographic clone ``k`` of +every FRS row at ``id + k x 10**9`` (the copy-id table's +``geographic_support`` field) before any copy layer runs. This module builds +such frames from synthetic sernums, with ids as ``frs_spine`` makes them +(``benunit_id`` sernum x 100 + b, ``person_id`` sernum x 1000 + p), and copies +households through the six copy layers in the spine's order: the SPI copy, +the income band donor, the high-value copies, the CGT support copies, the +incidence twin and the residential arms. Each layer picks its households by +source id alone, so a K=1 frame's copies are exactly its clone 0's in a K=3 +frame. ``rule="table"`` mints every copy through the copy-id table, as the +stages do; ``rule="retired"`` replays the rule the table replaced (ten to the +digit count of the largest id the layer sees), for comparison. + +:func:`assert_copy_ids_follow_lineage` checks a real stage's output: every new +household is matched to its root in the stage's input by lineage columns, never +by id arithmetic, and all five id columns must equal the root's plus the copy's +ordinal times the field's declared radix. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime import copy_id_table +from microcosm.frame import Frame + +__all__ = [ + "COPY_PATH_COLUMNS", + "GEOGRAPHIC_RADIX", + "IdTables", + "assert_copy_ids_follow_lineage", + "base_id_tables", + "clone_first_tables", + "copy_chain", + "copy_households", + "id_frame", +] + +GEOGRAPHIC_RADIX = copy_id_table.radix("geographic_support") +#: Each layer's ordinal on the household table, in the spine's order. +COPY_PATH_COLUMNS = ("spi", "hv", "cgt", "twin", "arm") +_PERSON_ID_COLUMNS = ("person_id", "person_household_id", "person_benunit_id") + + +@dataclass(frozen=True) +class IdTables: + """The five id columns of a UK frame, with each household's copy path.""" + + person: pd.DataFrame + benunit: pd.DataFrame + household: pd.DataFrame + + def ids(self, entity: str) -> np.ndarray: + column = { + "person": "person_id", + "benunit": "benunit_id", + "household": "household_id", + }[entity] + return getattr(self, entity)[column].to_numpy(dtype=np.int64) + + +def base_id_tables(n_households: int = 60) -> IdTables: + """Synthetic FRS ids: sernums 1..n, one to four people in one or two units.""" + + people: list[tuple[int, int, int]] = [] + benunits: list[int] = [] + for sernum in range(1, n_households + 1): + size = 1 + sernum % 4 + units = min(size, 1 + sernum % 2) + benunits.extend(sernum * 100 + b for b in range(1, units + 1)) + people.extend( + (sernum * 1000 + p, sernum, sernum * 100 + min(p, units)) + for p in range(1, size + 1) + ) + sernums = np.arange(1, n_households + 1, dtype=np.int64) + household = pd.DataFrame( + { + "household_id": sernums, + "source_household_id": sernums, + "household_clone_index": np.zeros(n_households, dtype=np.int64), + **{ + column: np.zeros(n_households, dtype=np.int64) + for column in COPY_PATH_COLUMNS + }, + } + ) + person = pd.DataFrame(people, columns=list(_PERSON_ID_COLUMNS)).astype("int64") + benunit = pd.DataFrame({"benunit_id": np.asarray(benunits, dtype=np.int64)}) + return IdTables(person=person, benunit=benunit, household=household) + + +def clone_first_tables(tables: IdTables, n_clones: int) -> IdTables: + """``n_clones`` geographic clones of every row, clone-major. + + Clone ``k`` takes every id plus ``k x 10**9`` and ``household_clone_index`` + ``k``; ``source_household_id`` stays the clone-0 id. + """ + + persons, benunits, households = ( + [tables.person], + [tables.benunit], + [tables.household], + ) + for k in range(1, n_clones): + person = tables.person.copy() + for column in _PERSON_ID_COLUMNS: + person[column] = copy_id_table.mint(person[column], "geographic_support", k) + benunit = tables.benunit.copy() + benunit["benunit_id"] = copy_id_table.mint( + benunit["benunit_id"], "geographic_support", k + ) + household = tables.household.copy() + household["household_id"] = copy_id_table.mint( + household["household_id"], "geographic_support", k + ) + household["household_clone_index"] = k + persons.append(person) + benunits.append(benunit) + households.append(household) + return IdTables( + person=pd.concat(persons, ignore_index=True), + benunit=pd.concat(benunits, ignore_index=True), + household=pd.concat(households, ignore_index=True), + ) + + +def id_frame(tables: IdTables, *, time_period: str = "2024") -> Frame: + """``tables`` as a UK national frame at unit household weights.""" + + from microcosm.build.uk_runtime.national_frame import uk_national_frame + + return uk_national_frame( + person=tables.person, + benunit=tables.benunit, + household=tables.household, + household_weights=np.ones(len(tables.household)), + time_period=time_period, + ) + + +def _largest_id(tables: IdTables) -> int: + return max( + int(tables.person[list(_PERSON_ID_COLUMNS)].to_numpy().max()), + int(tables.benunit["benunit_id"].max()), + int(tables.household["household_id"].max()), + ) + + +def copy_households( + tables: IdTables, + *, + field: str, + path_column: str, + ordinals: dict[int, Sequence[int]], + rule: str, +) -> IdTables: + """Append copies of households at each ordinal, as the stages mint them. + + ``ordinals`` maps a household's position to the ordinals it is copied at; + each copy carries its household's path with ``path_column`` set to the + ordinal. ``rule="table"`` mints through the copy-id table's ``field``; + ``rule="retired"`` offsets by ten to the digit count of the largest id + in ``tables``, computed once for the layer as the stages did. + """ + + if rule == "retired": + multiplier = 10 ** max(1, len(str(_largest_id(tables)))) + elif rule != "table": + raise ValueError(f"Unknown copy-id rule {rule!r}.") + household = tables.household + persons, benunits, households = [tables.person], [tables.benunit], [household] + for k in sorted({k for values in ordinals.values() for k in values}): + positions = [p for p, values in ordinals.items() if k in values] + copied = household.iloc[positions].copy() + ids = set(copied["household_id"].tolist()) + members = tables.person.loc[ + tables.person["person_household_id"].isin(ids) + ].copy() + units = tables.benunit.loc[ + tables.benunit["benunit_id"].isin(set(members["person_benunit_id"])) + ].copy() + + def minted(values: pd.Series, k: int = k) -> np.ndarray: + if rule == "table": + return copy_id_table.mint(values, field, k) + return values.to_numpy(dtype=np.int64) + k * multiplier + + for column in _PERSON_ID_COLUMNS: + members[column] = minted(members[column]) + units["benunit_id"] = minted(units["benunit_id"]) + copied["household_id"] = minted(copied["household_id"]) + copied[path_column] = k + persons.append(members) + benunits.append(units) + households.append(copied) + result = IdTables( + person=pd.concat(persons, ignore_index=True), + benunit=pd.concat(benunits, ignore_index=True), + household=pd.concat(households, ignore_index=True), + ) + for entity in ("person", "benunit", "household"): + if len(np.unique(result.ids(entity))) != len(result.ids(entity)): + raise AssertionError(f"Minted {entity} ids are not unique.") + return result + + +def copy_chain( + tables: IdTables, *, rule: str = "table", layers: int | None = None +) -> IdTables: + """The six copy layers in the spine's order, each picking by source id. + + ``layers`` stops after that many layers (the SPI copy and the band donor + count as two). The CGT layer gives one household eleven copies besides + its root, so its largest ordinal has two digits, and the twin of every + household whose source id is a multiple of six becomes two, four or + eight arms. + """ + + def spi_copies(row) -> tuple[int, ...]: + return (1,) if row.spi == 0 and row.source_household_id % 3 != 0 else () + + def band_donors(row) -> tuple[int, ...]: + return (2,) if row.spi == 0 and row.source_household_id % 3 == 0 else () + + def high_value(row) -> tuple[int, ...]: + return tuple(range(1, 6)) if row.source_household_id % 7 == 1 else () + + def cgt_support(row) -> tuple[int, ...]: + source = int(row.source_household_id) + if source % 4 != 2: + return () + return tuple(range(1, 12 if source == 2 else 2 + source % 3)) + + def twins(row) -> tuple[int, ...]: + return (1,) + + def arms(row) -> tuple[int, ...]: + source = int(row.source_household_id) + if row.twin != 1 or source % 6 != 0: + return () + return tuple(range(1, 2 ** (1 + (source // 6) % 3))) + + steps = ( + # The SPI copy of base-channel households, then the income band donors + # of the others, both in the spi_support_channel field. + ("spi_support_channel", "spi", spi_copies), + ("spi_support_channel", "spi", band_donors), + ("high_value_support", "hv", high_value), + ("cgt_support_split", "cgt", cgt_support), + ("cgt_incidence_clone", "twin", twins), + ("cgt_residential_split", "arm", arms), + ) + for field, column, ordinals_of in steps[:layers]: + ordinals = { + position: ordinals_of(row) + for position, row in enumerate(tables.household.itertuples(index=False)) + } + tables = copy_households( + tables, + field=field, + path_column=column, + ordinals={p: values for p, values in ordinals.items() if values}, + rule=rule, + ) + return tables + + +def _entity_tables( + source: Frame | Mapping[str, pd.DataFrame], +) -> Mapping[str, pd.DataFrame]: + if isinstance(source, Frame): + return {entity: source.table(entity) for entity in source.entities} + return source + + +def assert_copy_ids_follow_lineage( + before: Frame | Mapping[str, pd.DataFrame], + after: Frame | Mapping[str, pd.DataFrame], + *, + field: str, + key_columns: Sequence[str], + ordinal_column: str | None = None, +) -> int: + """Every new household is its root's copy at ``ordinal x radix(field)``. + + The root is the household of ``before`` whose ``key_columns`` equal the + copy's (the stages copy them unchanged), never one found from the copy's + id. The copy's ordinal is ``ordinal_column`` on the new row, or 1 for a + single-copy layer. Members pair up in person_id order, and each copy's + benefit units are its root's at the same offset. Ids are read as Python + integers, exact above 2**53. Returns the number of copies checked. + """ + + radix = copy_id_table.radix(field) + keys = list(key_columns) + before_tables, after_tables = _entity_tables(before), _entity_tables(after) + root_household = before_tables["household"] + root_keys = list(zip(*(root_household[c].tolist() for c in keys), strict=True)) + if len(set(root_keys)) != len(root_keys): + raise AssertionError("The lineage key does not name one root household.") + roots = dict(zip(root_keys, root_household["household_id"].tolist(), strict=True)) + household = after_tables["household"] + new = household.loc[~household["household_id"].isin(root_household["household_id"])] + ordinals = ( + [1] * len(new) if ordinal_column is None else new[ordinal_column].tolist() + ) + person = after_tables["person"] + root_person = before_tables["person"] + benunit_ids = set(after_tables["benunit"]["benunit_id"].tolist()) + copy_keys = zip(*(new[c].tolist() for c in keys), strict=True) + for copy, key, ordinal in zip( + new["household_id"].tolist(), copy_keys, ordinals, strict=True + ): + root = roots[key] + offset = int(ordinal) * radix + assert copy == root + offset, (copy, root, ordinal) + members = person.loc[person["person_household_id"] == copy] + root_members = root_person.loc[root_person["person_household_id"] == root] + assert len(members) == len(root_members) + for column in _PERSON_ID_COLUMNS: + assert sorted(members[column].tolist()) == sorted( + value + offset for value in root_members[column].tolist() + ), column + assert { + value + offset for value in root_members["person_benunit_id"].tolist() + } <= benunit_ids + return len(new) diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 6f3bb24827..13fa9c85d2 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -73,6 +73,7 @@ from microcosm.build.uk_runtime.rowwise_dataset import ( UK_SPINE_LINEAGE_COLUMNS, _refuse_preassigned_geography, + refuse_dense_clone_id_overflow, ) from microcosm.frame import engine_tables @@ -829,6 +830,16 @@ def _dry_run_plan( person_ids["person_benunit_id"], benunit_ids["benunit_id"], ) + # A plan must refuse what the build would (microcosm#931). + refuse_dense_clone_id_overflow( + n_clones=max((args.n_clones, *(args.candidate_clone_counts or ()))), + id_multiplier=id_multiplier, + ids=( + household["household_id"], + *(person_ids[column] for column in PERSON_ID_COLUMNS), + benunit_ids["benunit_id"], + ), + ) if args.source_lineage_modulus is not None: household = apply_uk_source_lineage_modulus( household, @@ -1186,6 +1197,16 @@ def _ladder_dry_run_plan( person_ids["person_benunit_id"], benunit_ids["benunit_id"], ) + # A plan must refuse what the build would (microcosm#931). + refuse_dense_clone_id_overflow( + n_clones=max((args.n_clones, *(args.candidate_clone_counts or ()))), + id_multiplier=id_multiplier, + ids=( + household["household_id"], + *(person_ids[column] for column in PERSON_ID_COLUMNS), + benunit_ids["benunit_id"], + ), + ) if args.source_lineage_modulus is not None: household = apply_uk_source_lineage_modulus( household, From 52da13c9737ae1b9661fed8e942a5fe174101428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:09:08 +0000 Subject: [PATCH 014/108] Follow up the review of the fixed copy-id radices (#931) Three reviews of 22bcdbe82 found texts that overstate or misstate what it does, and refusals and dense paths that no test drives. This commit corrects the texts and adds the tests. The identity tool's decoders and the other items ruled into 1b stay in the next commit. - Stage 46's notes said the geography identity kernel and the identity tool read the arm rule. The kernel reads lineage columns, and the tool decodes the retired rule until 1b, so the notes now name the executor only, as the module docstring does. source_stages.json is re-derived, and the coverage manifest's source digests and the H2 fixture's notes are regenerated. - The row walk the executor replaced read a row of only integer and float columns as float64; an all-integer row stays int64. The _source_lineage docstring and the changelog now say so, and the exactness test carries a float column, on which the replaced walk fails. - The dense refusal fires wherever the dense offset would overflow int64, as a spine with residential arms at multiples of 10**18 does at K >= 2. Its docstring, its message and the changelog now say that. require_below's message no longer says that only a later stage may fill the field. - The changelog says each copy row now sits at its parent's ids plus its layer's offset, where it used to sit a power of ten above the largest id its layer saw. It also says that national ids pass 2**53 from the twin on (and at stage 42 from ordinal 901), that only the new refusals print counts, that create_uk_spi_support_tables no longer takes id_multiplier, and that copy rows join across versions by lineage, not by build ids. - The CGT split's bound check moves into a helper so its boundary can be tested. The lineage test helper reports counts, never ids or keys. What moves: no value or id. Every node re-keys again, through the text in graph_kernels.py, copy_id_table.py and rowwise_dataset.py and the helper in cgt_support.py. Tests: - Stage 16 on clone-first K=1 and K=3 frames nests exactly. - The executor takes the declared radix where the frame's maximum would give another, names each stage's own field, and refuses parentless rows by count. - The id_remapping texts quote the table's radices. - Stages 26 and 42 mint from the immediate parent on frames with SPI copies in FRS id spaces. - frs_spine (end to end), stages 42, 43 and 46 and the replace path refuse incoming ids by count. - The CGT bound holds 10,000 rows and refuses 10,001. - A drifted SPI id_remapping is refused. - The copy-id binding set is derived from imports. - The dense refusal and the K=1 path run through the pool expand, the crosswalk route, both dry-run plans and the graph's uk.full.expand, on ids above 2**53. - The twin's negative-prior count is pinned. - H2 parity passes on the regenerated fixture. The 11 tests in test_uk_identity_stability_receipts still fail until 1b, as at 22bcdbe82. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-copy-id-table.changed.md | 6 +- .../uk/release_input_coverage_manifest.json | 50 +-- .../src/microcosm/build/uk/source_stages.json | 2 +- .../src/microcosm/build/uk/spec/sources.yaml | 2 +- .../microcosm/build/uk_runtime/cgt_support.py | 16 +- .../build/uk_runtime/copy_id_table.py | 7 +- .../build/uk_runtime/graph_kernels.py | 6 +- .../build/uk_runtime/rowwise_dataset.py | 24 +- .../uk/test_uk_cgt_residential_split.py | 20 ++ .../engine_free/uk/test_uk_cgt_structure.py | 20 +- .../engine_free/uk/test_uk_cgt_support.py | 117 ++++++- .../engine_free/uk/test_uk_copy_id_table.py | 328 +++++++++++++++++- .../tests/engine_free/uk/test_uk_frs_spine.py | 23 ++ .../uk/test_uk_full_population_graph.py | 82 +++++ .../tests/engine_free/uk/test_uk_graph.py | 68 +++- .../uk/test_uk_high_value_home_support.py | 72 ++++ .../uk/test_uk_ladder_rowwise_clone.py | 62 ++++ .../engine_free/uk/test_uk_rowwise_dry_run.py | 58 +++- .../tests/engine_free/uk/test_uk_spi_spine.py | 12 + .../engine_free/uk/test_uk_spi_support.py | 35 ++ .../parity/uk_spine/sources/fixture.json | 2 +- .../microcosm_build/uk_clone_first_frames.py | 74 +++- .../uk_ladder_rowwise_clone.py | 12 +- 23 files changed, 996 insertions(+), 102 deletions(-) diff --git a/changelog.d/931-uk-copy-id-table.changed.md b/changelog.d/931-uk-copy-id-table.changed.md index cd35a29ce2..6a70406387 100644 --- a/changelog.d/931-uk-copy-id-table.changed.md +++ b/changelog.d/931-uk-copy-id-table.changed.md @@ -1,6 +1,6 @@ The UK spine mints every copy row's ids from fixed decimal fields declared in `microcosm.build.uk_runtime.copy_id_table`, not from a power of ten above the largest id each copy layer sees (#931). Once the spine clones households at its start, that largest id would grow with the clone count K, so every layer's offset would move with K and the K=1 spine would stop being the clone-0 slice of the K=15 spine. Copy `k` in a field takes `id + k × radix` on all five id columns, so every copy id is a function of its source id and its copy path alone. The fields, least significant first: `vintage` (10 ordinals at 10^8, reserved for pooled FRS vintages, minted by nothing yet), `geographic_support` (100 at 10^9, for the clone-first spine), `spi_support_channel` (3 at 10^11: the SPI copy at 1, the income band donor at 2), `high_value_support` (6 at 10^12), `cgt_support_split` (10,000 at 10^13), `cgt_incidence_clone` (2 at 10^17) and `cgt_residential_split` (8 at 10^18, the top field). The largest mintable id is about 7.2e18, below int64's limit, which the module checks when it loads. -- **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 rather than row by row, which upcast an all-numeric row to float64. Refusals print counts, never ids. -- **What moves.** Base FRS rows keep their ids. Every copy row's five id columns change: SPI copies by +10^11, band donors by +2×10^11, high-value copies by k×10^12, CGT copies by k×10^13, twins by +10^17 and arms by j×10^18. Relative id order is unchanged at K=1, so sorts, ties, ranks and folds are unchanged. Values move on copy rows through every draw keyed on the copy's own id: the SPI income stage's second stage, stages 19 to 24 and 27 to 41, every twin's Advani-Summers prior (so the CGT gains layer is re-realised), stages 48 to 54 and policyengine-uk's Warm Home Discount draw on copy households in spine engine calls. Base rows move second-order through pooled steps: the high-value selection, the regional value ranks, the landlord walk, the residential solve, the anchor, the CGT walks and calibration. Draws keyed on `person_source_id` do not move. National ids now pass 2^53 from the CGT support split on. -- **Dense line.** The dense expand keeps its multiplier above the spine's largest id, which a table-minted spine's twins and arms leave no room above: a dense pool with more than one clone is refused at every entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) in one message naming #931 and the clone-first spine. Dense builds with K > 1 wait for #931's clone-first spine; K = 1, and a spine without copy rows, expand as before. +- **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 on any frame; the row-by-row walk it replaces read a row holding only integer and float columns as float64, inexact above 2^53. The new refusals print counts, never ids. `create_uk_spi_support_tables` no longer takes an `id_multiplier`. +- **What moves.** Base FRS rows keep their ids. Every copy row's five id columns now sit at its parent row's ids plus its layer's offset: SPI copies 10^11, band donors 2×10^11, high-value copy k k×10^12, CGT copy k k×10^13, twins 10^17 and arm j j×10^18. Each used to be a power of ten above the largest id its layer saw. Relative id order is unchanged at K=1, so sorts, ties, ranks and folds are unchanged. Values move on copy rows through every draw keyed on the copy's own id: the SPI income stage's second stage, stages 19 to 24 and 27 to 41, every twin's Advani-Summers prior (so the CGT gains layer is re-realised), stages 48 to 54 and policyengine-uk's Warm Home Discount draw on copy households in spine engine calls. Base rows move second-order through pooled steps: the high-value selection, the regional value ranks, the landlord walk, the residential solve, the anchor, the CGT walks and calibration. Draws keyed on `person_source_id` do not move. National ids pass 2^53 from the incidence twin on, and at the CGT support split for any copy at ordinal 901 or more. The released file's identifiers, ranks of the build ids, do not move at K=1, while its `*_build_id` columns carry the new copy ids, so joining copy rows across versions goes through lineage (source id and copy path), not build ids. +- **Dense line.** The dense expand keeps its multiplier above the spine's largest id. A spine carrying residential arms (multiples of 10^18), as a production national spine does, leaves no int64 room for a second dense clone, so every dense entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) refuses wherever the dense offset would overflow int64, in one message naming #931 and the clone-first spine. Dense builds with K > 1 of such a spine wait for #931's clone-first spine; K = 1, and a spine whose ids leave room (one without copy rows, say), expand as before. - **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes, and the SPI support, band-donor, high-value and CGT clone and anchor stages also bind the helper modules they call. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 1e48a779fa..e29c22561f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "f2c3e11d1fc3876dec0bcd91f5f27453851ac52ff4d989de76ab5ddbde46fd14", + "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index daf1f8ff80..e998b5ff26 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -7857,7 +7857,7 @@ "cgt_residential_probability", "capital_gains_residential_property" ], - "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time." + "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor reads for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time." }, { "stage": "hmrc_cgt_asset_type_spine", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 2797dd92d8..62b13af209 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -6384,7 +6384,7 @@ stages: nonnegative_outputs: - cgt_residential_probability - capital_gains_residential_property - notes: 'Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row''s stake: at the top of the gains distribution one stake is several times a band''s expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household''s ids and arm j taking every entity id plus j times the radix of the copy-id table''s top field (microcosm#931), the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone''s halves. Every household''s mass and composition and the total household mass are conserved; the receipt is gated at stage time.' + notes: 'Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row''s stake: at the top of the gains distribution one stake is several times a band''s expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household''s ids and arm j taking every entity id plus j times the radix of the copy-id table''s top field (microcosm#931), the lineage rule the executor reads for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone''s halves. Every household''s mass and composition and the total household mass are conserved; the receipt is gated at stage time.' - stage: hmrc_cgt_asset_type_spine survey: HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed source: https://www.gov.uk/government/statistics/capital-gains-tax-statistics diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index 8fcf89ace7..02cbce14fc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -436,6 +436,19 @@ def cgt_support_copy_counts( return np.maximum(counts, 1) +def _refuse_copies_beyond_the_copy_id_field(copies: np.ndarray) -> None: + """Refuse a household divided into more rows than its copy-id field holds. + + ``copies`` counts each household's rows, root included, so a household + of ``bound`` rows takes ordinals up to ``bound - 1`` and fits; one more + row does not. The refusal is the fixed :data:`CGT_SUPPORT_COPY_BOUND_REFUSAL`. + """ + + bound = copy_id_table.ordinal_bound(CGT_SUPPORT_COPY_ID_FIELD) + if len(copies) and int(np.max(copies)) > bound: + raise ValueError(CGT_SUPPORT_COPY_BOUND_REFUSAL) + + def split_cgt_support_households( frame: Frame, *, @@ -500,8 +513,7 @@ def split_cgt_support_households( copies[selected] = cgt_support_copy_counts( weights[selected], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT ) - if int(copies.max()) > copy_id_table.ordinal_bound(CGT_SUPPORT_COPY_ID_FIELD): - raise ValueError(CGT_SUPPORT_COPY_BOUND_REFUSAL) + _refuse_copies_beyond_the_copy_id_field(copies) for table, columns in ( (person, _PERSON_ID_COLUMNS), (benunit, ("benunit_id",)), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py index 5da3189426..4c31b6c1fe 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py @@ -181,7 +181,8 @@ def require_below(ids: object, field: str, *, what: str = "ids") -> None: """Refuse any id at or above ``field``'s radix, printing only the count. A stage that mints ``field`` copies rows whose ids must all sit below it, - so that every copy's parent is recoverable from the copy's id. + so that every copy's parent is recoverable from the copy's id; the raw + FRS ids sit below the first field, ``vintage``. """ values = _ids(ids, what=what) @@ -189,8 +190,8 @@ def require_below(ids: object, field: str, *, what: str = "ids") -> None: if reached: raise ValueError( f"{reached} of the {what} reach the {field!r} copy-id field " - f"(radix {radix(field)}), which only a later stage may fill " - "(microcosm#931)." + f"(radix {radix(field)}): ids from that radix up are reserved for " + "rows minted in that field or a later one (microcosm#931)." ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 2d207dac9a..a305198cf2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -1409,8 +1409,10 @@ def _source_lineage( ) -> pd.Series: """Derive new target-to-immediate-source ids from a structural result. - Vectorised on exact int64 ids: a row-by-row walk would upcast an - all-numeric row to float64, which is inexact above 2**53. The refusals + Vectorised on exact int64 ids, so it is exact on any frame. A row-by-row + walk reads a row whose columns hold only integers and floats as float64, + which is inexact above 2**53 (an all-integer row stays int64, and a row + with a string or boolean column comes back as objects). The refusals print counts, never an id. """ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index d81a85fad0..1372c08a00 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -101,12 +101,14 @@ def refuse_dense_clone_id_overflow( Dense clone ``c`` offsets every id by ``c`` times a multiplier above the spine's largest id. Since microcosm#931 the spine mints its copies' ids in - fixed fields of the copy-id table, up to the CGT incidence twins at - 10**17 and the residential arms at 10**18, so a production spine leaves no - room for a second clone above its ids. Dense builds with more than one - clone wait for the clone-first spine, which mints the geographic clones at - its start (#931 group (c)); one clone, and a spine without copy rows, - expand as before. The message names no id. + fixed fields of the copy-id table, the CGT incidence twins at 10**17 and + the residential arms at multiples of 10**18, so a spine carrying arms, as + a production national spine does, leaves no int64 room for a second + clone. This refuses wherever the dense offset would overflow int64; one + clone, and any spine whose ids leave room (one without copy rows, say), + expand as before. Dense builds with more than one clone of such a spine + wait for the clone-first spine, which mints the geographic clones at its + start (#931 group (c)). The message names no id. """ if type(n_clones) is not int or n_clones <= 1: @@ -120,11 +122,11 @@ def refuse_dense_clone_id_overflow( raise ValueError( f"Dense geographic clones of this spine at n_clones={n_clones} would " "overflow int64: since microcosm#931 the spine mints its copy ids in " - "the copy-id table's fixed fields, up to 10**18, and every dense " - "clone is offset above the spine's largest id. Dense builds with " - "more than one clone wait for the clone-first spine of microcosm#931, " - "which mints the geographic clones at its start; n_clones=1 still " - "expands." + "the copy-id table's fixed fields, with residential arms at " + "multiples of 10**18, and every dense clone is offset above the " + "spine's largest id. Dense builds with more than one clone of such a " + "spine wait for the clone-first spine of microcosm#931, which mints " + "the geographic clones at its start; n_clones=1 still expands." ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_residential_split.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_residential_split.py index 57bb7f5dbe..ba8fec85c6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_residential_split.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_residential_split.py @@ -40,6 +40,7 @@ from microcosm.frame import WeightKind from test_support.microcosm_build.uk_clone_first_frames import ( assert_copy_ids_follow_lineage, + with_last_household_moved, ) PARAMETERS = UKCGTPolicyParameters( @@ -283,6 +284,25 @@ def test_arms_take_their_households_ids_plus_j_times_the_declared_radix() -> Non assert checked == result.evidence()["arms_created"] == 4 +def test_split_refuses_incoming_ids_at_its_copy_id_radix_by_count() -> None: + plain = _frame( + [50_000.0, 80_000.0, 90_000.0, 0.0, 1_000.0, 0.0], + weights=[10.0, 20.0, 30.0], + people_per_household=2, + ) + frame = with_last_household_moved(plain, radix("cgt_residential_split")) + + with pytest.raises( + ValueError, + match=r"^2 of the incoming person_household_id values reach the " + r"'cgt_residential_split' copy-id field", + ) as caught: + split_cgt_residential_households( + frame, facts=_facts(frame), parameters=PARAMETERS + ) + assert str(2 + radix("cgt_residential_split")) not in str(caught.value) + + def test_split_refuses_too_many_gainers_an_already_split_or_classified_frame() -> None: gains = [10_000.0] * (CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS + 1) frame = _frame(gains, people_per_household=len(gains)) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_structure.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_structure.py index b6d6cf318e..36bf724628 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_structure.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_structure.py @@ -41,6 +41,7 @@ from test_support.microcosm_build.uk_cgt_support import support_frame from test_support.microcosm_build.uk_clone_first_frames import ( assert_copy_ids_follow_lineage, + with_last_household_moved, ) @@ -161,10 +162,11 @@ def test_clone_splits_exact_mass_and_uses_oldest_adult_carriers() -> None: ) assert result.carrier_count == 2 # The priors key on each twin's own person_id, minted at the copy-id - # table's cgt_incidence_clone radix (microcosm#931); the receipt counts - # the negative ones the draw kept. + # table's cgt_incidence_clone radix (microcosm#931): on this synthetic + # frame both carriers draw a negative prior (one did under the retired + # ids), and the receipt counts the negative ones the draw kept. + assert result.negative_prior_count == 2 assert result.negative_prior_count == int((gainers.capital_gains < 0.0).sum()) - assert (gainers.capital_gains < 0.0).any() def test_twins_take_their_originals_ids_plus_the_declared_radix() -> None: @@ -189,6 +191,18 @@ def test_twins_take_their_originals_ids_plus_the_declared_radix() -> None: assert checked == 3 +def test_clone_refuses_incoming_ids_at_its_copy_id_radix_by_count() -> None: + frame = with_last_household_moved(_adult_frame(), radix("cgt_incidence_clone")) + + with pytest.raises( + ValueError, + match=r"^2 of the incoming person_household_id values reach the " + r"'cgt_incidence_clone' copy-id field", + ) as caught: + clone_cgt_incidence(frame, distribution=_distribution()) + assert str(2 + radix("cgt_incidence_clone")) not in str(caught.value) + + def test_prior_spline_keeps_negative_values_and_extrapolates_linearly() -> None: draws = _draw_banded_priors( np.zeros(4), diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index c1e7ebb791..0a7377817d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -35,6 +35,7 @@ HOUSEHOLD_IS_CGT_SUPPORT_COPY, UKCGTSupportSplitStageTransform, _assert_cgt_support_split_stage_parameters, + _refuse_copies_beyond_the_copy_id_field, cgt_support_copy_counts, cgt_support_household_wealth, cgt_support_income_band, @@ -43,7 +44,7 @@ select_cgt_support_households, split_cgt_support_households, ) -from microcosm.build.uk_runtime.copy_id_table import radix +from microcosm.build.uk_runtime.copy_id_table import ordinal_bound, radix from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_INCOME_BAND_LOWER_BOUNDS, load_hmrc_cgt_joint_distribution, @@ -61,6 +62,7 @@ ) from test_support.microcosm_build.uk_clone_first_frames import ( assert_copy_ids_follow_lineage, + with_last_household_moved, ) # --- the reference scenario ------------------------------------------------- @@ -569,6 +571,85 @@ def test_split_copies_take_their_roots_ids_plus_k_times_the_declared_radix(): assert checked == result.copies_created == 5 +def _in_frs_id_spaces(frame): + """``frame`` in FRS-style id spaces, its SPI-channel rows as SPI copies. + + Households keep their ids, benefit units take household x 100 + slot and + people household x 1000 + member, so no two id columns share values; an + SPI-channel household and its members and units move up 10**11, as the + SPI copies sit after stage 16. + """ + + person = frame.table("person").copy() + benunit = frame.table("benunit").copy() + household = frame.table("household").copy() + spi = household["household_support_channel"].eq("spi").to_numpy() + shift = pd.Series( + np.where(spi, radix("spi_support_channel"), 0), + index=household["household_id"].to_numpy(), + ) + sernum = person["person_household_id"] + offset = sernum.map(shift) + member = person.groupby("person_household_id").cumcount() + 1 + slot = ( + person.groupby("person_household_id")["person_benunit_id"] + .rank(method="dense") + .astype("int64") + ) + units = dict( + zip(person["person_benunit_id"], sernum * 100 + slot + offset, strict=True) + ) + person["person_id"] = sernum * 1000 + member + offset + person["person_benunit_id"] = person["person_benunit_id"].map(units) + person["person_household_id"] = sernum + offset + benunit["benunit_id"] = benunit["benunit_id"].map(units) + household["household_id"] = household["household_id"] + shift.to_numpy() + # Group tables stay sorted by id, so the SPI copies follow the base rows. + order = np.argsort(household["household_id"].to_numpy(), kind="stable") + return uk_national_frame( + person=person, + benunit=benunit.sort_values("benunit_id", ignore_index=True), + household=household.iloc[order].reset_index(drop=True), + household_weights=np.asarray(frame.weights_for("household").values)[order], + weight_kind=frame.weights_for("household").kind, + time_period="2024", + ) + + +def test_split_copies_of_spi_rows_take_their_parents_ids_plus_k_times_the_radix(): + # Separate id spaces, and a split household that is an SPI copy at + # 10**11: a stage that minted one id column from another, or from the + # SPI copy's root, would miss here (microcosm#931). + frame = _in_frs_id_spaces( + _scenario_frame( + ages=[(45,), (50, 30, 8), (45,), (45,), (45,), (60, 58), (45,)], + benunits=[(0,), (0, 1, 1), (0,), (0,), (0,), (0, 1), (0,)], + ) + ) + result = split_cgt_support_households( + frame, + distribution=support_distribution(SCENARIO_TOP_CELLS), + parameters=PARAMETERS, + headroom=SCENARIO_HEADROOM, + ) + + checked = assert_copy_ids_follow_lineage( + frame, + result.frame, + field="cgt_support_split", + key_columns=("council_tax",), + ordinal_column=CGT_SUPPORT_COPY_INDEX_COLUMN, + ) + assert checked == result.copies_created == 5 + household = result.frame.table("household") + copies = household.loc[household[HOUSEHOLD_IS_CGT_SUPPORT_COPY], "household_id"] + # Household 2, on the SPI channel, is among the split: its copies sit at + # 2 + 10**11 + k x 10**13. + assert {2 + radix("spi_support_channel") + k * COPY_RADIX for k in (1, 2)} <= set( + copies.tolist() + ) + + def test_split_refuses_a_household_beyond_the_copy_id_field_with_a_fixed_message(): # Id 2 would need 10,001 copies of at most 60, one more than the # cgt_support_split field's ordinals hold (microcosm#931). The refusal is @@ -584,6 +665,40 @@ def test_split_refuses_a_household_beyond_the_copy_id_field_with_a_fixed_message assert "10,001" not in str(caught.value) +def test_a_household_of_exactly_the_fields_bound_in_rows_fits(): + # A weight of exactly 60 x 10,000 divides into 10,000 rows, root + # included, so its largest copy ordinal is 9,999: the field holds it, and + # one more row does not (microcosm#931). + bound = ordinal_bound("cgt_support_split") + copies = cgt_support_copy_counts( + [CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * bound, 1.0], + CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + ) + assert copies.tolist() == [bound, 1] + _refuse_copies_beyond_the_copy_id_field(copies) + _refuse_copies_beyond_the_copy_id_field(np.zeros(0, dtype="int64")) + with pytest.raises(ValueError) as caught: + _refuse_copies_beyond_the_copy_id_field(copies + np.asarray([1, 0])) + assert str(caught.value) == CGT_SUPPORT_COPY_BOUND_REFUSAL + + +def test_split_refuses_incoming_ids_at_its_copy_id_radix_by_count(): + frame = with_last_household_moved(_scenario_frame(), radix("cgt_support_split")) + + with pytest.raises( + ValueError, + match=r"^1 of the incoming person_household_id values reach the " + r"'cgt_support_split' copy-id field", + ) as caught: + split_cgt_support_households( + frame, + distribution=support_distribution(SCENARIO_TOP_CELLS), + parameters=PARAMETERS, + headroom=SCENARIO_HEADROOM, + ) + assert str(7 + radix("cgt_support_split")) not in str(caught.value) + + def test_split_flags_only_copies_and_writes_the_family_copy_count(): result = _scenario_split() household = result.frame.table("household").set_index("household_id") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py index 24924f6e1a..7aac59350c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -12,6 +12,7 @@ import ast import inspect import json +import re from pathlib import Path import numpy as np @@ -19,7 +20,14 @@ import pytest import microcosm.build as build -from microcosm.build.uk_runtime import copy_id_table, graph_kernels +from microcosm.build.uk_runtime import ( + cgt_residential_split, + cgt_structure, + cgt_support, + copy_id_table, + graph_kernels, + high_value_home_support, +) from microcosm.build.uk_runtime.cgt_residential_split import ( CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS, ) @@ -40,18 +48,27 @@ ) from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE from microcosm.build.uk_runtime.rowwise_dataset import ( + clone_uk_dataset_tables_with_rowwise_geography, expand_uk_geographic_pool, refuse_dense_clone_id_overflow, ) from microcosm.build.uk_runtime.spi_band_donors import ( SPI_INCOME_BAND_DONOR_CLONE_INDEX, + SPI_INCOME_BAND_DONOR_ID_REMAPPING, +) +from microcosm.build.uk_runtime.spi_support import ( + SPI_SUPPORT_ID_REMAPPING, + create_uk_spi_support_tables, + support_channel_column, ) from test_support.microcosm_build.uk_clone_first_frames import ( COPY_PATH_COLUMNS, GEOGRAPHIC_RADIX, + IdTables, base_id_tables, clone_first_tables, copy_chain, + copy_households, id_frame, ) @@ -155,6 +172,65 @@ def test_a_malformed_table_is_refused() -> None: radix("clone_index") +def _declared_id_remapping(operations, kind: str) -> str: + return next( + parameters["id_remapping"] + for operation_kind, parameters in operations + if operation_kind == kind + ) + + +@pytest.mark.parametrize( + ("field", "remapping", "ordinal_text"), + [ + ("spi_support_channel", SPI_SUPPORT_ID_REMAPPING, "1 x the field's radix"), + ( + "spi_support_channel", + SPI_INCOME_BAND_DONOR_ID_REMAPPING, + f"{SPI_INCOME_BAND_DONOR_CLONE_INDEX} x the field's radix", + ), + ( + high_value_home_support.HIGH_VALUE_SUPPORT_COPY_ID_FIELD, + _declared_id_remapping( + high_value_home_support.high_value_support_operation_parameters(), + high_value_home_support.HIGH_VALUE_SUPPORT_OPERATION_KIND, + ), + "k x the field's radix", + ), + ( + cgt_support.CGT_SUPPORT_COPY_ID_FIELD, + _declared_id_remapping( + cgt_support.cgt_support_split_operation_parameters(), + "split_top_wealth_support_households", + ), + f"k up to {ordinal_bound('cgt_support_split') - 1:,}", + ), + ( + cgt_structure.CGT_CLONE_COPY_ID_FIELD, + cgt_structure.CGT_CLONE_ID_REMAPPING, + "the field's radix", + ), + ( + cgt_residential_split.CGT_RESIDENTIAL_COPY_ID_FIELD, + _declared_id_remapping( + cgt_residential_split.cgt_residential_split_operation_parameters(), + "split_liable_gainers_by_residential_probability", + ), + "j x the field's radix", + ), + ], +) +def test_each_declared_id_remapping_restates_its_fields_radix( + field, remapping, ordinal_text +) -> None: + # The manifests restate these strings verbatim (closed-world checks); the + # numbers they quote must be the table's, so neither goes stale alone. + assert remapping.startswith(f"copy-id table field {field}") + powers = [int(power) for power in re.findall(r"\(10\*\*(\d+)\)", remapping)] + assert [10**power for power in powers] == [radix(field)] + assert ordinal_text in remapping + + # --- the helpers ---------------------------------------------------------------- @@ -233,6 +309,14 @@ def test_the_frs_spine_refuses_raw_ids_that_reach_the_first_copy_field() -> None def test_k1_ids_sort_as_the_retired_rule_sorted_them_and_base_ids_keep() -> None: + """The table orders ids at K=1 as the retired rule did. + + This catches a table whose fields run out of stage order below the twin, + but not every misordering: swapping the twin and arm fields leaves every + K=1 sort unchanged, because arms are copies of twins. The exact pin of + the fields (``test_the_fields_bounds_and_radices_are_the_declared_ones``) + and each stage's ``require_below`` carry that order. + """ base = base_id_tables() table = copy_chain(base, rule="table") retired = copy_chain(base, rule="retired") @@ -302,6 +386,69 @@ def test_clone_zero_and_its_copies_keep_the_k1_ids_on_clone_first_frames() -> No ) +def _spi_support_on_clone_first_tables(base: IdTables, n_clones: int): + """The real SPI support stage on ``n_clones`` geographic clones of ``base``. + + Every clone of the same base households is selected, so only the stage's + id minting can tell K=1 from K=3 apart. + """ + + tables = clone_first_tables(base, n_clones) + sources = [ + int(source) + for source in base.household["household_id"].tolist() + if source % 3 != 0 + ] + return create_uk_spi_support_tables( + person=tables.person, + benunit=tables.benunit, + household=tables.household[ + ["household_id", "source_household_id", "household_clone_index"] + ].assign(household_weight=1.0), + spi_household_count=None, + selected_household_ids=tuple( + source + clone * GEOGRAPHIC_RADIX + for clone in range(n_clones) + for source in sources + ), + source_year=2024, + ) + + +def test_the_spi_stage_mints_clone_zero_copies_at_their_k1_ids() -> None: + """Stage 16, where the retired rule grew with K, run on clone-first frames. + + Clone 0's rows and SPI copies keep the K=1 ids exactly, and clone ``k``'s + are K=1's plus ``k x 10**9``. An offset sized from the ids the stage sees + would put K=3's copies a digit higher. The later copy stages' selections, + copy counts and draws move with K until group (a)'s later commits and (b) + make them per clone, so (b) owns their stage-level nesting tests. + """ + + base = base_id_tables(12) + k1 = _spi_support_on_clone_first_tables(base, 1) + k3 = _spi_support_on_clone_first_tables(base, 3) + channel = support_channel_column("household") + assert int((k1.household[channel] == "spi").sum()) > 0 + for entity, column in ( + ("household", "household_id"), + ("benunit", "benunit_id"), + ("person", "person_id"), + ("person", "person_household_id"), + ("person", "person_benunit_id"), + ): + k1_ids = getattr(k1, entity)[column].to_numpy(dtype=np.int64) + k3_ids = getattr(k3, entity)[column].to_numpy(dtype=np.int64) + np.testing.assert_array_equal( + np.sort(k3_ids), + np.sort( + np.concatenate( + [k1_ids + clone * GEOGRAPHIC_RADIX for clone in range(3)] + ) + ), + ) + + # --- the executor --------------------------------------------------------------- @@ -333,22 +480,132 @@ def test_the_executor_recovers_each_copy_parent_at_the_declared_radix( ) -def test_the_executor_stays_exact_above_2_53_on_all_integer_tables() -> None: - # The person table carries integer columns only, which a row-by-row walk - # would have read back as float64. +@pytest.mark.parametrize( + ("stage", "field", "layers", "path_column"), + [ + # No SPI layer below: the frame's largest id has five digits. + ("high_value_home_support", "high_value_support", 0, "hv"), + # SPI copies but no high-value copies: twelve digits, not thirteen. + ("cgt_support_split", "cgt_support_split", 2, "cgt"), + ], +) +def test_the_executor_takes_the_declared_radix_where_the_frame_maximum_differs( + stage, field, layers, path_column +) -> None: base = base_id_tables() - before = id_frame(copy_chain(base, layers=5)) - after = id_frame(copy_chain(base)) + before_tables = copy_chain(base, layers=layers) + after_tables = copy_households( + before_tables, + field=field, + path_column=path_column, + ordinals={0: (1, 2), 7: (1,), 11: (3,)}, + rule="table", + ) + before = id_frame(before_tables) + largest = max(int(before_tables.ids(entity).max()) for entity in before.entities) + assert 10 ** len(str(largest)) != radix(field) + offset = graph_kernels._declared_copy_radix(stage, before) + assert offset == radix(field) + for entity in before.entities: + lineage = graph_kernels._source_lineage( + before, id_frame(after_tables), entity, id_offset=offset + ) + targets = lineage.index.to_numpy(dtype=np.int64) + assert len(targets) > 0 + np.testing.assert_array_equal(lineage.to_numpy(), parent(targets, field)) + + +def test_the_executor_names_each_stages_own_copy_id_field() -> None: + assert dict(graph_kernels._COPY_ID_FIELDS) == { + "high_value_home_support": high_value_home_support.HIGH_VALUE_SUPPORT_COPY_ID_FIELD, + "cgt_support_split": cgt_support.CGT_SUPPORT_COPY_ID_FIELD, + "cgt_incidence_clone": cgt_structure.CGT_CLONE_COPY_ID_FIELD, + "cgt_residential_split": cgt_residential_split.CGT_RESIDENTIAL_COPY_ID_FIELD, + } + + +def test_the_executor_stays_exact_above_2_53_on_integer_and_float_tables() -> None: + # The person table holds integer ids and one float column and nothing + # else, so a row-by-row walk (iterrows) reads each row as float64, inexact + # above 2**53: the retired walk named parents that are not incumbents here. + # The vectorised read is exact. + base = base_id_tables() + + def with_income(tables: IdTables) -> IdTables: + person = tables.person.assign(employment_income=1_500.5) + return IdTables( + person=person, benunit=tables.benunit, household=tables.household + ) + + before = id_frame(with_income(copy_chain(base, layers=5))) + after = id_frame(with_income(copy_chain(base))) + person = after.table("person") + assert set(person.dtypes.map(lambda dtype: dtype.kind)) == {"i", "f"} lineage = graph_kernels._source_lineage( before, after, "person", id_offset=radix("cgt_residential_split") ) targets = lineage.index.to_numpy(dtype=np.int64) assert int(targets.min()) > 2**53 + assert lineage.dtype == np.dtype("int64") np.testing.assert_array_equal( lineage.to_numpy(), targets % radix("cgt_residential_split") ) +def test_the_executor_refuses_new_rows_without_an_incumbent_parent_by_count() -> None: + base = base_id_tables() + before_tables = copy_chain(base, layers=2) + stray = 10**12 + 99_999 + after_tables = copy_households( + before_tables, + field="high_value_support", + path_column="hv", + ordinals={0: (1,)}, + rule="table", + ) + after_tables = IdTables( + person=pd.concat( + [ + after_tables.person, + pd.DataFrame( + { + "person_id": [10**12 + 99_999_001], + "person_household_id": [stray], + "person_benunit_id": [10**12 + 9_999_901], + } + ), + ], + ignore_index=True, + ), + benunit=pd.concat( + [after_tables.benunit, pd.DataFrame({"benunit_id": [10**12 + 9_999_901]})], + ignore_index=True, + ), + household=pd.concat( + [ + after_tables.household, + after_tables.household.iloc[[0]].assign(household_id=stray), + ], + ignore_index=True, + ), + ) + before, after = id_frame(before_tables), id_frame(after_tables) + with pytest.raises( + ValueError, match=r"^UK stage produced 1 new 'household' row\(s\) whose id" + ) as caught: + graph_kernels._source_lineage( + before, after, "household", id_offset=radix("high_value_support") + ) + assert str(stray) not in str(caught.value) + # Without a declared radix or a source-id column, new rows are refused by + # count too. + with pytest.raises( + ValueError, match=r"^UK stage produced 2 new 'household' row\(s\) without" + ) as caught: + graph_kernels._source_lineage(before, after, "household", id_offset=None) + assert str(stray) not in str(caught.value) + + def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: base = base_id_tables() before = id_frame(copy_chain(base, layers=4)) @@ -414,10 +671,52 @@ def _pool_inputs(tables): } -def test_dense_clones_of_a_table_minted_spine_are_refused_in_one_message() -> None: +def _crosswalk_clone(*, n_clones, person, benunit, household, source_year, **_): + """The crosswalk clone route, on a one-area England crosswalk.""" + + crosswalk = pd.DataFrame( + [ + { + "oa_code": "E0001", + "lsoa_code": "E0101", + "msoa_code": "E0201", + "la_code": "E06000063", + "constituency_code": "E14000001", + "region_code": "E12000007", + "country": "England", + "population": 75, + } + ] + ) + result = clone_uk_dataset_tables_with_rowwise_geography( + person=person, + benunit=benunit, + household=household, + crosswalk=crosswalk, + n_clones=n_clones, + source_year=source_year, + require_all_countries=False, + ) + return result.household, result.person + + +def _pool_expand(*, n_clones, **inputs): + """The dense expand both the ladder clone route and uk.full.expand run.""" + + pool = expand_uk_geographic_pool(n_clones=n_clones, **inputs) + return pool.frame.table("household"), pool.frame.table("person") + + +@pytest.mark.parametrize("route", [_pool_expand, _crosswalk_clone]) +def test_dense_clones_of_a_table_minted_spine_are_refused_in_one_message( + route, +) -> None: + # The row-wise tool's two dry-run plans are refused the same way + # (test_uk_rowwise_dry_run.py and test_uk_ladder_rowwise_clone.py), and + # the graph's uk.full.expand in test_uk_full_population_graph.py. spine = copy_chain(base_id_tables()) with pytest.raises(ValueError) as caught: - expand_uk_geographic_pool(n_clones=2, **_pool_inputs(spine)) + route(n_clones=2, **_pool_inputs(spine)) message = str(caught.value) assert "microcosm#931" in message assert "clone-first spine" in message @@ -429,16 +728,15 @@ def test_dense_clones_of_a_table_minted_spine_are_refused_in_one_message() -> No ) -def test_one_dense_clone_of_a_table_minted_spine_still_expands() -> None: +@pytest.mark.parametrize("route", [_pool_expand, _crosswalk_clone]) +def test_one_dense_clone_of_a_table_minted_spine_still_expands(route) -> None: spine = copy_chain(base_id_tables()) - pool = expand_uk_geographic_pool(n_clones=1, **_pool_inputs(spine)) - np.testing.assert_array_equal( - pool.frame.table("household")["household_id"].to_numpy(), - spine.ids("household"), - ) + household, person = route(n_clones=1, **_pool_inputs(spine)) + assert household["household_id"].dtype == np.dtype("int64") np.testing.assert_array_equal( - pool.frame.table("person")["person_id"].to_numpy(), spine.ids("person") + household["household_id"].to_numpy(), spine.ids("household") ) + np.testing.assert_array_equal(person["person_id"].to_numpy(), spine.ids("person")) def test_a_spine_without_copy_rows_still_expands_into_dense_clones() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index 70f0a881e7..a3d3d2c7a4 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -1687,6 +1687,29 @@ def test_frs_spine_refuses_an_adult_tab_without_sppadj(tmp_path: Path) -> None: build_uk_frs_spine_frame(tmp_path, stage=stage) +def test_frs_spine_refuses_raw_ids_that_reach_the_first_copy_id_field( + tmp_path: Path, +) -> None: + """A synthetic sernum of 100,000 puts its person ids (sernum x 1000 + p) + at the copy-id table's first radix, 10**8, which the copy layers' fields + sit above; the spine refuses it by count, never naming the id + (microcosm#931).""" + + tables = _fixture_tables() + for rows in tables.values(): + for row in rows: + if row.get("SERNUM") == 2: + row["SERNUM"] = 100_000 + stage = _write_fixture(tmp_path, tables) + + with pytest.raises( + ValueError, + match=r"^1 of the raw FRS person_id values reach the 'vintage' copy-id field", + ) as caught: + build_uk_frs_spine_frame(tmp_path, stage=stage) + assert "100000001" not in str(caught.value) + + @pytest.mark.parametrize( ("job", "sic", "expected"), [ diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py index 6abbf82f5a..cb858e4f01 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py @@ -82,6 +82,88 @@ def test_legacy_population_graph_preserves_rows_weights_geography_and_replays( assert replay.population("uk.full.expand").mass_log == actual.mass_log +#: A residential arm plus a twin of the copy-id table (microcosm#931): every +#: synthetic id moved here sits above 2**53 and leaves the dense multiplier +#: (ten to the spine's digit count) above int64. +_TOP_FIELD_OFFSET = 10**18 + 10**17 +_ID_COLUMNS = { + "person": ("person_id", "person_household_id", "person_benunit_id"), + "benunit": ("benunit_id",), + "household": ("household_id",), +} + + +def _top_field_source_frame(): + frame = source_frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for entity, columns in _ID_COLUMNS.items(): + for column in columns: + tables[entity][column] = ( + tables[entity][column].astype("int64") + _TOP_FIELD_OFFSET + ) + return type(frame)( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _run_legacy(k, path, store): + graph, registry = graph_and_registry(k, geography_assignment="legacy") + return run_graph( + compile_graph(graph), + sources={"uk_ladder": path, "fixture": path}, + store=store, + kernels=registry, + ) + + +def test_one_dense_clone_of_a_spine_at_the_top_copy_fields_expands_exactly( + monkeypatch, toy_ladder, tmp_path +): + """uk.full.expand at K=1 on ids in the copy-id table's top fields. + + The multiplier is above int64 there, and the expand kernel multiplies no + empty column by it; every id comes through as exact int64, equal to the + unshifted run's plus the offset. K=2 is refused naming microcosm#931, + never an id (microcosm#931; dense K > 1 waits for the clone-first spine). + """ + from microcosm.graph import NodeRejectedError + from test_support.microcosm_build import uk_full_population_graph + + _, path = toy_ladder + plain = _run_legacy(1, path, ContentStore(tmp_path / "plain")) + monkeypatch.setattr( + uk_full_population_graph, "source_frame", _top_field_source_frame + ) + moved = _run_legacy(1, path, ContentStore(tmp_path / "moved")) + # Every population node of the graph: the sample, its normalisation and + # the expand (the later nodes carry cells and artifacts). + for node in ("uk.full.sample", "uk.full.normalize", "uk.full.expand"): + before, after = plain.population(node), moved.population(node) + for entity, columns in _ID_COLUMNS.items(): + for column in columns: + values = after.table(entity)[column] + assert values.dtype == np.dtype("int64"), (node, column) + np.testing.assert_array_equal( + values.to_numpy(), + before.table(entity)[column].to_numpy(dtype=np.int64) + + _TOP_FIELD_OFFSET, + ) + assert ( + int(moved.population("uk.full.expand").table("person")["person_id"].min()) + > 2**53 + ) + with pytest.raises(NodeRejectedError, match="clone-first spine") as caught: + _run_legacy(2, path, ContentStore(tmp_path / "refused")) + assert "microcosm#931" in str(caught.value) + smallest = int(_top_field_source_frame().table("household")["household_id"].min()) + assert str(smallest) not in str(caught.value) + + def test_k_changes_expansion_but_never_sampling(): first, _ = graph_and_registry(1) second, _ = graph_and_registry(3) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 75052ee434..f763d29bd4 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -311,36 +311,76 @@ def changed_helper_bytes(path: Path, helper=helper_path, read=original): #: channel's clone helper (``spi_support``), which reads the copy-id table; #: every other minting stage's module reads the table itself (microcosm#931). _SPI_COPY_ID_STAGES = frozenset({"spi_support_channel", "spi_income_band_donors"}) +#: Stages that bind ``spi_support`` or ``frs_spine``, which read the copy-id +#: table, for names that never mint, decode or bound a copy id: constants, +#: the support channel's column names and source fill, and frs_spine's tab +#: reader and self-employment rule. They need not bind the table. +_COPY_ID_TABLE_NAME_ONLY_STAGES = frozenset( + {"company_cars", "property_components", "spi_benefit_coherence"} +) +#: The functions those stages may import from the two helpers. +_COPY_ID_TABLE_FREE_HELPER_FUNCTIONS = frozenset( + { + "fill_support_channel_from_source", + "read_pinned_tab", + "support_channel_column", + "support_source_id_column", + "uc_gainful_self_employment", + } +) + + +def _names_imported_from(module, helpers: set[str]) -> set[str]: + import ast + import inspect + + tree = ast.parse(Path(inspect.getfile(module)).read_text(encoding="utf-8")) + return { + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + and node.module + and node.module.rsplit(".", 1)[-1] in helpers + for alias in node.names + } def test_every_stage_that_mints_or_bounds_copy_ids_binds_the_copy_id_table() -> None: - """``source_hash`` does not follow imports, so every stage whose module - reads the copy-id table, and the two SPI support stages, whose minting - runs in spi_support, bind it; no other stage does (microcosm#931).""" + """``source_hash`` does not follow imports, so every stage whose module or + bound helpers read the copy-id table binds it, save the stages that bind + a reading helper for table-free names alone (microcosm#931).""" from microcosm.build.uk_runtime import copy_id_table, graph_kernels, spi_support - assert "copy_id_table" in _imported_module_names(spi_support) reading = { stage for stage in graph_kernels._STAGE_MODULES - if "copy_id_table" in _imported_module_names(graph_kernels._stage_module(stage)) - } - assert reading == { - "frs_spine", - "high_value_home_support", - "cgt_support_split", - "cgt_incidence_clone", - "cgt_incidence_anchor", - "cgt_residential_split", + if any( + "copy_id_table" in _imported_module_names(module) + for module in ( + graph_kernels._stage_module(stage), + *graph_kernels._STAGE_HELPER_MODULES.get(stage, ()), + ) + ) } binding = { stage for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() if copy_id_table in helpers } - assert binding == reading | _SPI_COPY_ID_STAGES + assert _COPY_ID_TABLE_NAME_ONLY_STAGES <= reading + assert binding == reading - _COPY_ID_TABLE_NAME_ONLY_STAGES + assert _SPI_COPY_ID_STAGES <= binding for stage in _SPI_COPY_ID_STAGES: assert spi_support in graph_kernels._STAGE_HELPER_MODULES[stage] + for stage in _COPY_ID_TABLE_NAME_ONLY_STAGES: + module = graph_kernels._stage_module(stage) + assert "copy_id_table" not in _imported_module_names(module), stage + functions = { + name + for name in _names_imported_from(module, {"spi_support", "frs_spine"}) + if not name.isupper() + } + assert functions <= _COPY_ID_TABLE_FREE_HELPER_FUNCTIONS, stage def test_copy_id_table_changes_invalidate_every_stage_that_reads_it(monkeypatch): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py index 25429cdb82..7fe8207f06 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py @@ -137,6 +137,78 @@ def test_copies_take_their_roots_ids_plus_k_times_the_declared_radix() -> None: assert checked == result.copies_created == 4 * (hv.HIGH_VALUE_SUPPORT_COPIES - 1) +def _frame_with_spi_copies(): + """The owner frame in FRS id spaces, with SPI copies of its top owners. + + Households keep their sernum, benefit units take sernum x 100 + 1 and + people sernum x 1000 + 1. Each of the four most valuable owners also has + an SPI copy at every id plus 10**11, whose provenance names its root, so + a copy's immediate parent and its provenance differ. ``marker`` names + each row: lineage never leans on an id or on the provenance column. + """ + base = _frame() + household = base.table("household").copy() + benunit = base.table("benunit").copy() + person = base.table("person").copy() + benunit["benunit_id"] = benunit["benunit_id"] * 100 + 1 + person["person_benunit_id"] = person["person_benunit_id"] * 100 + 1 + person["person_id"] = person["person_id"] * 1000 + 1 + household["marker"] = np.arange(1, len(household) + 1) + spi = 10**11 + top = household.loc[household["tenure_type"] != "RENT_PRIVATELY"].tail(4) + copies = top.assign( + household_id=top["household_id"] + spi, + household_support_channel=pd.array(["spi"] * len(top), dtype="string"), + household_support_clone_index=1, + marker=top["marker"] + 1_000, + ) + members = person.loc[person["person_household_id"].isin(top["household_id"])] + member_units = benunit.loc[benunit["benunit_id"].isin(members["person_benunit_id"])] + return uk_national_frame( + person=pd.concat( + [ + person, + members.assign( + person_id=members["person_id"] + spi, + person_household_id=members["person_household_id"] + spi, + person_benunit_id=members["person_benunit_id"] + spi, + ), + ], + ignore_index=True, + ), + benunit=pd.concat( + [benunit, member_units.assign(benunit_id=member_units["benunit_id"] + spi)], + ignore_index=True, + ), + household=pd.concat([household, copies], ignore_index=True), + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=np.full(len(household) + len(copies), 10.0), + ) + + +def test_copies_of_spi_copies_take_their_parents_ids_plus_k_times_the_radix() -> None: + # Separate id spaces and SPI copies, whose provenance differs from their + # immediate parent: a stage that minted from the provenance column, or one + # id column from another, would miss here (microcosm#931). + frame = _frame_with_spi_copies() + result = hv.split_high_value_support_households(frame, stratum_lower=0.9) + + checked = assert_copy_ids_follow_lineage( + frame, + result.frame, + field=hv.HIGH_VALUE_SUPPORT_COPY_ID_FIELD, + key_columns=("marker",), + ordinal_column=HIGH_VALUE_SUPPORT_INDEX_COLUMN, + ) + assert checked == result.copies_created > 0 + household = result.frame.table("household") + copied = household.loc[household[HIGH_VALUE_SUPPORT_COPY_COLUMN]] + # Some copies are of SPI copies, and some of base households. + assert (copied["marker"] > 1_000).any() + assert (copied["marker"] < 1_000).any() + + def test_the_split_refuses_incoming_ids_at_its_copy_id_radix() -> None: frame = _frame() household = frame.table("household").copy() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_ladder_rowwise_clone.py index 8343214bfd..f504691c62 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_ladder_rowwise_clone.py @@ -459,6 +459,68 @@ def test_driver_ladder_dry_run_matches_real_assignment( assert manifest["rowwise_dataset"]["source_lineage"]["explicit"] is None +def test_driver_ladder_dry_run_refuses_dense_clones_of_a_table_minted_spine( + monkeypatch, toy_ladder, tmp_path +): + """A plan refuses what the build would (microcosm#931). + + Every synthetic id sits at a residential arm plus a twin of the copy-id + table, so a second dense clone has no int64 room; one clone plans. + """ + pytest.importorskip("tables") + pytest.importorskip("h5py") + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + offset = 10**18 + 10**17 + person = _person_frame() + for column in ("person_id", "person_household_id", "person_benunit_id"): + person[column] = person[column] + offset + input_h5 = tmp_path / "staging.h5" + _write_seam_h5( + input_h5, + household=_household_frame().assign( + household_id=lambda table: table["household_id"] + offset + ), + person=person, + benunit=_benunit_frame().assign( + benunit_id=lambda table: table["benunit_id"] + offset + ), + ) + + def argv(n_clones: int, out: str) -> list[str]: + return [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--input-sha256", + builder._sha256(input_h5), + "--ladder", + str(ladder_path), + "--ladder-sha256", + builder._sha256(ladder_path), + "--n-clones", + str(n_clones), + "--seed", + "7", + "--out", + str(tmp_path / out), + "--dry-run", + ] + + monkeypatch.setattr(sys, "argv", argv(2, "refused")) + with pytest.raises( + ValueError, match="clone-first spine of microcosm#931" + ) as caught: + builder.main() + assert str(offset + 11) not in str(caught.value) + assert not (tmp_path / "refused" / builder.DRY_RUN_PLAN_FILENAME).exists() + monkeypatch.setattr(sys, "argv", argv(1, "planned")) + assert builder.main() == 0 + assert (tmp_path / "planned" / builder.DRY_RUN_PLAN_FILENAME).exists() + + def test_driver_ladder_candidate_k_matches_independent_single_k_plan( monkeypatch, toy_ladder, tmp_path ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dry_run.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dry_run.py index 8ec8de6442..7693029547 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dry_run.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dry_run.py @@ -104,6 +104,8 @@ def _write_toy_h5( path: Path, *, household: pd.DataFrame | None = None, + person: pd.DataFrame | None = None, + benunit: pd.DataFrame | None = None, fmt: str = "table", ) -> None: with pd.HDFStore(path) as store: @@ -113,8 +115,18 @@ def _write_toy_h5( format=fmt, data_columns=fmt == "table", ) - store.put("person", _person_frame(), format=fmt, data_columns=fmt == "table") - store.put("benunit", _benunit_frame(), format=fmt, data_columns=fmt == "table") + store.put( + "person", + _person_frame() if person is None else person, + format=fmt, + data_columns=fmt == "table", + ) + store.put( + "benunit", + _benunit_frame() if benunit is None else benunit, + format=fmt, + data_columns=fmt == "table", + ) store.put( "time_period", pd.Series(["2023"]), @@ -274,6 +286,48 @@ def test_driver_dry_run_writes_plan_only(monkeypatch, tmp_path) -> None: assert plan["source_lineage"]["explicit"] is None +def test_driver_dry_run_refuses_dense_clones_of_a_table_minted_spine( + monkeypatch, tmp_path +) -> None: + """A plan refuses what the build would (microcosm#931). + + Every synthetic id sits at a residential arm plus a twin of the copy-id + table, so a second dense clone has no int64 room. + """ + pytest.importorskip("tables") + pytest.importorskip("h5py") + builder = _load_builder_module() + offset = 10**18 + 10**17 + person = _person_frame() + for column in ("person_id", "person_household_id", "person_benunit_id"): + person[column] = person[column] + offset + input_h5 = tmp_path / "populace_uk_2023.h5" + crosswalk_path = tmp_path / "crosswalk.csv.gz" + output_dir = tmp_path / "out" + _write_toy_h5( + input_h5, + household=_household_frame().assign( + household_id=lambda table: table["household_id"] + offset + ), + person=person, + benunit=_benunit_frame().assign( + benunit_id=lambda table: table["benunit_id"] + offset + ), + ) + _crosswalk_frame().to_csv(crosswalk_path, index=False) + + monkeypatch.setattr( + sys, "argv", _dry_run_argv(input_h5, output_dir, crosswalk_path) + ) + with pytest.raises( + ValueError, match="clone-first spine of microcosm#931" + ) as caught: + builder.main() + assert "n_clones=2" in str(caught.value) + assert str(offset + 1001) not in str(caught.value) + assert not (output_dir / builder.DRY_RUN_PLAN_FILENAME).exists() + + def test_crosswalk_dry_run_reports_candidate_k_rows_bytes_and_support( monkeypatch, tmp_path ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py index 3e17aeb58f..7e200cb557 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py @@ -1009,6 +1009,18 @@ def test_support_stage_parameters_refuse_gate_declaration_drift() -> None: _support_stage_parameters(stage, seed=42) +def test_support_stage_parameters_refuse_id_remapping_drift() -> None: + # The manifest restates the copy-id table's spi_support_channel rule + # (microcosm#931); a drifted restatement is refused. + stage = _with_mutated_operation( + _committed_stage("spi_support_channel"), + "stack_zero_weight_donors", + id_remapping="__drift__", + ) + with pytest.raises(ValueError, match="id_remapping drifted"): + _support_stage_parameters(stage, seed=42) + + def test_readers_of_the_support_record_take_the_declared_allocation_reason() -> None: """microcosm#1069 c6: the stratified sentence, not the 50% constant.""" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py index fb7348f38d..9d609775b0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py @@ -626,6 +626,41 @@ def test_exact_mass_correction_handles_one_ulp_bounce() -> None: assert weights.total == 0.6 +def test_replace_spi_support_refuses_a_base_whose_ids_reach_the_spi_field() -> None: + # The rebuilt copies mint in the spi_support_channel field, so a surviving + # base household already at or above its radix (here a synthetic id at a + # high-value copy's offset) is refused by count, never given colliding + # ids (microcosm#931). + dead = _certified_like_dead_spi_support() + person, benunit, household = ( + dead.person.copy(), + dead.benunit.copy(), + dead.household.copy(), + ) + shift = 10**12 + members = person["person_household_id"] == 2 + units = set(person.loc[members, "person_benunit_id"]) + household.loc[household["household_id"] == 2, "household_id"] += shift + person.loc[members, ["person_id", "person_household_id", "person_benunit_id"]] += ( + shift + ) + benunit.loc[benunit["benunit_id"].isin(units), "benunit_id"] += shift + + with pytest.raises( + ValueError, + match=r"^1 of the incoming household_id values reach the " + r"'spi_support_channel' copy-id field", + ) as caught: + replace_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household, + seed=7, + source_year=2023, + ) + assert str(2 + shift) not in str(caught.value) + + def test_replace_spi_support_refuses_to_discard_positive_spi_mass() -> None: dead = _certified_like_dead_spi_support() household = dead.household.copy() diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 3e15f4398b..9abc56aa01 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -483,7 +483,7 @@ "cgt_residential_probability", "capital_gains_residential_property" ], - "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time.", + "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor reads for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time.", "operations": [ { "artifact_role": "cgt_asset_type_facts", diff --git a/test_support/microcosm_build/uk_clone_first_frames.py b/test_support/microcosm_build/uk_clone_first_frames.py index 101f1ddab3..4774d981cf 100644 --- a/test_support/microcosm_build/uk_clone_first_frames.py +++ b/test_support/microcosm_build/uk_clone_first_frames.py @@ -13,10 +13,11 @@ stages do; ``rule="retired"`` replays the rule the table replaced (ten to the digit count of the largest id the layer sees), for comparison. -:func:`assert_copy_ids_follow_lineage` checks a real stage's output: every new -household is matched to its root in the stage's input by lineage columns, never -by id arithmetic, and all five id columns must equal the root's plus the copy's -ordinal times the field's declared radix. +:func:`assert_copy_ids_follow_lineage` checks a stage's output on a synthetic +frame: every new household is matched to its root in the stage's input by +lineage columns, never by id arithmetic, and all five id columns must equal the +root's plus the copy's ordinal times the field's declared radix. Its failures +print counts, never an id or a key. """ from __future__ import annotations @@ -40,6 +41,7 @@ "copy_chain", "copy_households", "id_frame", + "with_last_household_moved", ] GEOGRAPHIC_RADIX = copy_id_table.radix("geographic_support") @@ -144,6 +146,31 @@ def id_frame(tables: IdTables, *, time_period: str = "2024") -> Frame: ) +def with_last_household_moved(frame: Frame, offset: int) -> Frame: + """``frame`` with its last household's id, and its members', up ``offset``. + + For the stages' incoming-id refusals: the moved household reaches a copy + field its stage must find empty. Weights, strata and the mass log are + kept; only ``household_id`` and the members' ``person_household_id`` + move. + """ + + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + household, person = tables["household"], tables["person"] + last = int(household["household_id"].iloc[-1]) + household.loc[household.index[-1], "household_id"] = last + offset + members = person["person_household_id"] == last + person.loc[members, "person_household_id"] = last + offset + return Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + def _largest_id(tables: IdTables) -> int: return max( int(tables.person[list(_PERSON_ID_COLUMNS)].to_numpy().max()), @@ -294,7 +321,9 @@ def assert_copy_ids_follow_lineage( id. The copy's ordinal is ``ordinal_column`` on the new row, or 1 for a single-copy layer. Members pair up in person_id order, and each copy's benefit units are its root's at the same offset. Ids are read as Python - integers, exact above 2**53. Returns the number of copies checked. + integers, exact above 2**53. For synthetic frames: a failure counts the + copies off each rule and names no id or key. Returns the number of + copies checked. """ radix = copy_id_table.radix(field) @@ -314,20 +343,35 @@ def assert_copy_ids_follow_lineage( root_person = before_tables["person"] benunit_ids = set(after_tables["benunit"]["benunit_id"].tolist()) copy_keys = zip(*(new[c].tolist() for c in keys), strict=True) + failures = { + "without a root": 0, + "off the root's household id": 0, + "off the root's members": 0, + "off the root's benefit units": 0, + } for copy, key, ordinal in zip( new["household_id"].tolist(), copy_keys, ordinals, strict=True ): - root = roots[key] + root = roots.get(key) + if root is None: + failures["without a root"] += 1 + continue offset = int(ordinal) * radix - assert copy == root + offset, (copy, root, ordinal) + if copy != root + offset: + failures["off the root's household id"] += 1 members = person.loc[person["person_household_id"] == copy] root_members = root_person.loc[root_person["person_household_id"] == root] - assert len(members) == len(root_members) - for column in _PERSON_ID_COLUMNS: - assert sorted(members[column].tolist()) == sorted( - value + offset for value in root_members[column].tolist() - ), column - assert { - value + offset for value in root_members["person_benunit_id"].tolist() - } <= benunit_ids + if len(members) != len(root_members) or any( + sorted(members[column].tolist()) + != sorted(value + offset for value in root_members[column].tolist()) + for column in _PERSON_ID_COLUMNS + ): + failures["off the root's members"] += 1 + root_units = root_members["person_benunit_id"].tolist() + if not {value + offset for value in root_units} <= benunit_ids: + failures["off the root's benefit units"] += 1 + problems = {rule: count for rule, count in failures.items() if count} + assert not problems, ( + f"Of {len(new)} {field!r} copies, these many break each rule: {problems}." + ) return len(new) diff --git a/test_support/microcosm_build/uk_ladder_rowwise_clone.py b/test_support/microcosm_build/uk_ladder_rowwise_clone.py index de863de2c3..0e3875ab29 100644 --- a/test_support/microcosm_build/uk_ladder_rowwise_clone.py +++ b/test_support/microcosm_build/uk_ladder_rowwise_clone.py @@ -174,13 +174,19 @@ def _load_builder_module(): return module -def _write_seam_h5(path, *, household: pd.DataFrame | None = None) -> None: +def _write_seam_h5( + path, + *, + household: pd.DataFrame | None = None, + person: pd.DataFrame | None = None, + benunit: pd.DataFrame | None = None, +) -> None: from microcosm.build.uk_runtime import write_uk_national_frame from microcosm.build.uk_runtime.national_frame import uk_national_frame dataset = uk_national_frame( - person=_person_frame(), - benunit=_benunit_frame(), + person=_person_frame() if person is None else person, + benunit=_benunit_frame() if benunit is None else benunit, household=_household_frame() if household is None else household, time_period="2023", weight_kind=WeightKind.IMPORTANCE, From c1723a7104c54bf4bbc33a70ec341aec3d62f9df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:28:24 +0000 Subject: [PATCH 015/108] Decode UK copy ids at the declared radices outside the stages (#931) 22bcdbe82 and its follow-up 2f0f8ce86 moved every mint site, the executor and the anchor onto the copy-id table's fixed radices. Two decoders outside the stages still used the retired rule. The identity tool read the CGT support copies and the residential arms at ten to the digit count of the largest pre-split id. The dataset-size evaluation multiplied by a dense multiplier of 10**19, which a one-clone expansion of a table-minted spine records, and overflowed int64. This commit moves the identity tool onto the table and both decoders onto exact integers. It also makes the copy path's refusals count-only, adds static guards against the retired rule, and binds the shared keyed-draw helpers into every stage that draws through them. - Identity tool: _support_copy_lineage and _residential_arm_lineage take each copy's ordinal and parent at its field's declared radix (copy_id_table.ordinal and parent). They read the household table only, as exact int64; a non-integer id column is refused. Each receipt's id_multiplier now records the declared radix (10**13 for the support split, 10**18 for the arms). Before any receipt, the tool refuses a spine whose copy ids disagree with their lineage columns under the table, as every spine built before the table does. The refusal is one count-only message saying to run the tool at the commit that built the spine. The refusal and the decode are two small functions, so a retired-rule path could replace them and nothing else. - Size evaluation: dense ids decode back to spine ids exactly, at the multiplier the expansion recorded (the new id_multiplier argument of weight_tables), else the spine's, else the clone-0 ids'. 10**19 is included. A clone offset that int64 cannot hold, a non-integer id column and a multiplier that does not fit the dataset are refused by count. A dense file that keeps its true sources computes its spine stretch again; it used to fail on int(None). - Count-only refusals: - rowwise_geography: _remap_ids' overflow (it printed the multiplier and the largest id), clone_entity_frame's and assign_household_geography's duplicate checks (up to five ids), and the crosswalk route's check for duplicate household ids before cloning; - rowwise_dataset: its uniqueness and person-link checks; - spi_support: its uniqueness and person-link checks, the donor's duplicate and missing-source checks, and the selected-household check; - spi_spine: the stage-14 retained-leaf check; - cgt_structure: the CGT carrier check; - cgt_imputation: the redraw's missing-weight check. - Guards: only the dense line's own modules name id_multiplier_for_values until (c), each with its reason: rowwise_geography, rowwise_dataset, the package re-export, size_evaluation and the row-wise tool. Every module that mints, bounds or decodes copy ids reads the table, the identity tool among them, and names no retired rule. - Bindings: every spine stage whose module or bound helpers import microcosm.build.stochastic_assignment now binds it. A derived test holds the set, and a helper-only edit to it re-keys all 36 such stages and no control. No dense node calls it. - Texts: the stage-46 notes and the residential split's docstring say the identity tool reads the arm rule too. The changelog's binding sentence names only the helpers that mint or weigh copies (spi_support, cgt_support, rowwise_geography). What moves: no data value or id. Stage keys re-key on 31 stages that newly bind stochastic_assignment: frs_spine, frs_take_up, frs_person_draws, frs_household_draws, frs_brma, frs_annual_income_spells, spi_income_band_donors, hmrc_spi_income_spine, spi_housing_shell, was_wealth, was_lisa, property_components, property_purchases, nts_bus_travel, lcfs_consumption, etb_vat, etb_services, uc_reporter_redraw, spi_benefit_coherence, uc_capital_coherence, uc_take_up, uc_legacy_closure_take_up, pension_credit_take_up, child_benefit_take_up, uc_deduction_attributes, cgt_incidence_clone, cgt_incidence_anchor, salary_sacrifice, student_loans, isa_subscriptions and childcare_copy_take_up. Every spine node re-keys anyway through graph_kernels.py, as do the dense nodes that hash rowwise_dataset.py. Regenerated: source_stages.json (the stage-46 notes line), the coverage manifest's 25 source digests and the H2 fixture's notes line in fixture.json; uk_spine.json is unchanged. Tests: the 11 identity-tool tests that failed at 22bcdbe82 and 2f0f8ce86 pass again. The hand-built split frame is minted through copy_id_table; the other fixtures come from the real stages. New tests cover: - the old-spine refusal, through main and directly; - the arm decoder above 2**53; - a support copy at ordinal 9,999, which a float read misplaces; - the size evaluation's recorded, spine and clone-0 paths at 10**19, an older dense run at its own multiplier, the int64 refusal and the source-column stretch; - the count-only messages, with no id in them; - the executor's household_source_id branch; - the two static guards and the binding test. Synthetic mutations make each guard fail. The parent's weight_tables fails the four new size tests (OverflowError, TypeError). The 114 touched engine-free test files pass. The H2 fixture regenerates byte-identically, and H2 parity and the engine graph and ladder-clone tests pass in the engine slot. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-copy-id-table.changed.md | 6 +- .../uk/release_input_coverage_manifest.json | 50 ++-- .../src/microcosm/build/uk/source_stages.json | 2 +- .../src/microcosm/build/uk/spec/sources.yaml | 2 +- .../build/uk_runtime/cgt_imputation.py | 4 +- .../build/uk_runtime/cgt_residential_split.py | 2 +- .../build/uk_runtime/cgt_structure.py | 4 +- .../build/uk_runtime/graph_kernels.py | 92 ++++-- .../build/uk_runtime/rowwise_dataset.py | 56 ++-- .../build/uk_runtime/rowwise_geography.py | 41 +-- .../build/uk_runtime/size_evaluation.py | 114 +++++-- .../microcosm/build/uk_runtime/spi_spine.py | 4 +- .../microcosm/build/uk_runtime/spi_support.py | 52 ++-- .../engine_free/uk/test_uk_cgt_support.py | 4 +- .../engine_free/uk/test_uk_copy_id_table.py | 148 +++++++-- .../tests/engine_free/uk/test_uk_graph.py | 84 ++++++ .../uk/test_uk_identity_stability_receipts.py | 227 +++++++++++++- .../engine_free/uk/test_uk_rowwise_dataset.py | 19 +- .../uk/test_uk_rowwise_geography.py | 50 ++++ .../engine_free/uk/test_uk_size_evaluation.py | 168 +++++++++++ .../tests/engine_free/uk/test_uk_spi_spine.py | 5 +- .../engine_free/uk/test_uk_spi_support.py | 55 +++- .../parity/uk_spine/sources/fixture.json | 2 +- tools/verify_uk_identity_stability.py | 282 +++++++++++------- 24 files changed, 1166 insertions(+), 307 deletions(-) diff --git a/changelog.d/931-uk-copy-id-table.changed.md b/changelog.d/931-uk-copy-id-table.changed.md index 6a70406387..10fb12ef16 100644 --- a/changelog.d/931-uk-copy-id-table.changed.md +++ b/changelog.d/931-uk-copy-id-table.changed.md @@ -1,6 +1,6 @@ The UK spine mints every copy row's ids from fixed decimal fields declared in `microcosm.build.uk_runtime.copy_id_table`, not from a power of ten above the largest id each copy layer sees (#931). Once the spine clones households at its start, that largest id would grow with the clone count K, so every layer's offset would move with K and the K=1 spine would stop being the clone-0 slice of the K=15 spine. Copy `k` in a field takes `id + k × radix` on all five id columns, so every copy id is a function of its source id and its copy path alone. The fields, least significant first: `vintage` (10 ordinals at 10^8, reserved for pooled FRS vintages, minted by nothing yet), `geographic_support` (100 at 10^9, for the clone-first spine), `spi_support_channel` (3 at 10^11: the SPI copy at 1, the income band donor at 2), `high_value_support` (6 at 10^12), `cgt_support_split` (10,000 at 10^13), `cgt_incidence_clone` (2 at 10^17) and `cgt_residential_split` (8 at 10^18, the top field). The largest mintable id is about 7.2e18, below int64's limit, which the module checks when it loads. -- **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 on any frame; the row-by-row walk it replaces read a row holding only integer and float columns as float64, inexact above 2^53. The new refusals print counts, never ids. `create_uk_spi_support_tables` no longer takes an `id_multiplier`. +- **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 on any frame; the row-by-row walk it replaces read a row holding only integer and float columns as float64, inexact above 2^53. Refusals on the copy path print counts, never ids: the new ones, and the older checks that listed up to five ids or printed the largest (uniqueness, links, sources, carriers and weights in the clone helpers, the SPI support and spine stages, the row-wise dataset and the CGT stages). `create_uk_spi_support_tables` no longer takes an `id_multiplier`. The identity tool (`tools/verify_uk_identity_stability.py`) decodes the CGT support copies and the residential arms at their fields' declared radices and records the declared radix as each receipt's `id_multiplier`; it refuses, by count, a spine whose copy ids do not follow the table, which every spine built before the table is, and such a spine is receipted with the tool at the commit that built it. - **What moves.** Base FRS rows keep their ids. Every copy row's five id columns now sit at its parent row's ids plus its layer's offset: SPI copies 10^11, band donors 2×10^11, high-value copy k k×10^12, CGT copy k k×10^13, twins 10^17 and arm j j×10^18. Each used to be a power of ten above the largest id its layer saw. Relative id order is unchanged at K=1, so sorts, ties, ranks and folds are unchanged. Values move on copy rows through every draw keyed on the copy's own id: the SPI income stage's second stage, stages 19 to 24 and 27 to 41, every twin's Advani-Summers prior (so the CGT gains layer is re-realised), stages 48 to 54 and policyengine-uk's Warm Home Discount draw on copy households in spine engine calls. Base rows move second-order through pooled steps: the high-value selection, the regional value ranks, the landlord walk, the residential solve, the anchor, the CGT walks and calibration. Draws keyed on `person_source_id` do not move. National ids pass 2^53 from the incidence twin on, and at the CGT support split for any copy at ordinal 901 or more. The released file's identifiers, ranks of the build ids, do not move at K=1, while its `*_build_id` columns carry the new copy ids, so joining copy rows across versions goes through lineage (source id and copy path), not build ids. -- **Dense line.** The dense expand keeps its multiplier above the spine's largest id. A spine carrying residential arms (multiples of 10^18), as a production national spine does, leaves no int64 room for a second dense clone, so every dense entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) refuses wherever the dense offset would overflow int64, in one message naming #931 and the clone-first spine. Dense builds with K > 1 of such a spine wait for #931's clone-first spine; K = 1, and a spine whose ids leave room (one without copy rows, say), expand as before. -- **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes, and the SPI support, band-donor, high-value and CGT clone and anchor stages also bind the helper modules they call. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. +- **Dense line.** The dense expand keeps its multiplier above the spine's largest id. A spine carrying residential arms (multiples of 10^18), as a production national spine does, leaves no int64 room for a second dense clone, so every dense entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) refuses wherever the dense offset would overflow int64, in one message naming #931 and the clone-first spine. Dense builds with K > 1 of such a spine wait for #931's clone-first spine; K = 1, and a spine whose ids leave room (one without copy rows, say), expand as before. The dataset-size evaluation decodes dense ids back to spine ids on exact integers, at the multiplier the expansion recorded (`weight_tables(id_multiplier=)`), the spine's or the one the clone-0 ids give, 10^19 included, and refuses by count a dataset its multiplier does not fit; a dense file that keeps its true sources decodes its spine stretch again instead of failing. +- **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes, and the SPI support, band-donor, high-value and CGT clone and anchor stages also bind the helpers that mint or weigh their copies (`spi_support`, `cgt_support`, `rowwise_geography`). Every stage whose module or bound helpers draw through `microcosm.build.stochastic_assignment` now binds it, so an edit to the keyed draws re-keys each stage they move. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index e29c22561f..7a15e60db8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "77021dba8c14267add2e0dc29fb3f0a917ad08083f430ed2f374e07e478ccfca", + "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index e998b5ff26..e669ee5f94 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -7857,7 +7857,7 @@ "cgt_residential_probability", "capital_gains_residential_property" ], - "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor reads for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time." + "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time." }, { "stage": "hmrc_cgt_asset_type_spine", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 62b13af209..c43258c8ca 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -6384,7 +6384,7 @@ stages: nonnegative_outputs: - cgt_residential_probability - capital_gains_residential_property - notes: 'Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row''s stake: at the top of the gains distribution one stake is several times a band''s expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household''s ids and arm j taking every entity id plus j times the radix of the copy-id table''s top field (microcosm#931), the lineage rule the executor reads for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone''s halves. Every household''s mass and composition and the total household mass are conserved; the receipt is gated at stage time.' + notes: 'Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row''s stake: at the top of the gains distribution one stake is several times a band''s expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household''s ids and arm j taking every entity id plus j times the radix of the copy-id table''s top field (microcosm#931), the lineage rule the executor and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone''s halves. Every household''s mass and composition and the total household mass are conserved; the receipt is gated at stage time.' - stage: hmrc_cgt_asset_type_spine survey: HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed source: https://www.gov.uk/government/statistics/capital-gains-tax-statistics diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index 360c4d6fb3..8c74c2752f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -1766,8 +1766,8 @@ def impute_uk_capital_gains_with_report( ) if missing_households: raise ValueError( - "Person rows reference households with no weight: " - f"{sorted(missing_households)[:5]}." + f"Person rows reference {len(missing_households)} household(s) with " + "no weight." ) person_weight = ( person["person_household_id"].map(weights_by_household).to_numpy(dtype=float) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py index 2a6e4f2c60..cb38dc94c0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py @@ -27,7 +27,7 @@ total household mass are conserved; arm ``j`` takes every entity id plus ``j`` times the declared radix of the copy-id table's top field, ``cgt_residential_split`` (microcosm#931), the lineage rule the executor -reads for the support split and the clone too. +and the identity tool read for the support split and the clone too. """ from __future__ import annotations diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py index 0498f1de67..0d123feafc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py @@ -718,8 +718,8 @@ def _oldest_adult_indices( missing_households = household_ids - set(candidates.person_household_id) if missing_households: raise ValueError( - "Every CGT household requires an adult carrier; missing household " - f"id(s): {sorted(missing_households)[:5]}." + "Every CGT household requires an adult carrier; " + f"{len(missing_households)} household(s) have none." ) candidates["_row"] = candidates.index candidates = candidates.sort_values( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index a305198cf2..9cf94c2346 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -172,61 +172,79 @@ # Imported modules are not traversed by ``source_hash``. Bind relationship # helpers and the adapter's input-retention checks into every consuming stage. +# Every stage whose module or bound helpers draw through the shared keyed-draw +# helpers (``microcosm.build.stochastic_assignment``) binds them, so an edit to +# how a uniform is keyed or a binary is placed re-keys every stage it moves +# (microcosm#931); test_uk_graph derives the set from the imports. _STAGE_HELPER_MODULES = { # The three FRS draw stages' receipts measure their take-up flags through # frs_take_up's signal measurement and count support by the shared # disclosure rules (microcosm#1095). - "frs_take_up": (receipt_counts,), - "frs_person_draws": (frs_take_up, receipt_counts), - "frs_household_draws": (frs_take_up, receipt_counts), + "frs_take_up": (receipt_counts, stochastic_assignment), + "frs_person_draws": (frs_take_up, receipt_counts, stochastic_assignment), + "frs_household_draws": (frs_take_up, receipt_counts, stochastic_assignment), "frs_hmrc_spine_leaves": (frs_hmrc_source,), # The annual income descriptors' receipt follows the shared disclosure rules # (microcosm#1164). - "frs_annual_income_spells": (receipt_counts,), + "frs_annual_income_spells": (receipt_counts, stochastic_assignment), # The SPI income stage's imputation, band-donor resample and receipt live # in spi_income, so an edit there moves the stage's hash (microcosm#1278), # and its twin transport in spi_rank_transport (microcosm#840). - "hmrc_spi_income_spine": (spi_income, spi_rank_transport), + "hmrc_spi_income_spine": (spi_income, spi_rank_transport, stochastic_assignment), # The spine's raw ids sit below the copy-id table's first field, and every # copy layer mints its ids in its own field of the table; the two SPI # support stages mint through the support channel's clone helper, and the # high-value split weighs its copies through the CGT split's exact-total # correction (microcosm#931). The executor reads each id-offset stage's # declared radix from the table too. - "frs_spine": (uc_relationships, copy_id_table), + "frs_spine": (uc_relationships, copy_id_table, stochastic_assignment), "spi_support_channel": (spi_support, copy_id_table), - "spi_income_band_donors": (spi_support, copy_id_table), + "spi_income_band_donors": (spi_support, copy_id_table, stochastic_assignment), "high_value_home_support": (cgt_support, copy_id_table), "cgt_support_split": (copy_id_table,), - "cgt_incidence_clone": (rowwise_geography, copy_id_table), - "cgt_incidence_anchor": (rowwise_geography, copy_id_table), + "cgt_incidence_clone": (rowwise_geography, copy_id_table, stochastic_assignment), + "cgt_incidence_anchor": (rowwise_geography, copy_id_table, stochastic_assignment), "cgt_residential_split": (copy_id_table,), "frs_legacy_proxies": (*_ENGINE_MODULES,), "frs_education_grant_split": (*_ENGINE_MODULES,), - "frs_brma": (*_ENGINE_MODULES,), - "was_wealth": (*_ENGINE_MODULES,), + "frs_brma": (*_ENGINE_MODULES, stochastic_assignment), + "was_wealth": (*_ENGINE_MODULES, stochastic_assignment), # The LISA stage reads its household donor through the WAS cleaning and its # household predictors through the WAS recipient surface; the anchor counts # HMRC's payers through the typed ISA facts (microcosm#1415). - "was_lisa": (*_ENGINE_MODULES, was_wealth, hmrc_isa_statistics), + "was_lisa": ( + *_ENGINE_MODULES, + was_wealth, + hmrc_isa_statistics, + stochastic_assignment, + ), # Landlords' receipts read the vendored PRIS rows through their typed # loader, the tape through the income stage's preparation and uprating, # and the support channel's names (microcosm#1106). - "property_components": (hmrc_property_rental, spi_income, spi_support), + "property_components": ( + hmrc_property_rental, + spi_income, + spi_support, + stochastic_assignment, + ), # The regional values rank as the dense node does and read its knots # (microcosm#1112). "regional_home_values": (property_localisation, rowwise_geography), # The purchases read the vendored HMRC Stamp Tax rows through the feed-pinned # reader and mark SDLT's regions from the FRS region map (microcosm#1095). - "property_purchases": (ledger_fact_vendoring, rowwise_geography), - "nts_bus_travel": (*_ENGINE_MODULES, bus_use_incidence), + "property_purchases": ( + ledger_fact_vendoring, + rowwise_geography, + stochastic_assignment, + ), + "nts_bus_travel": (*_ENGINE_MODULES, bus_use_incidence, stochastic_assignment), # The energy level and prices are energy_pricing's (microcosm#1418): a # code-only change to them must re-key the stage (the independent review # of microcosm#1420, finding 5). - "lcfs_consumption": (*_ENGINE_MODULES, energy_pricing), - "etb_vat": (*_ENGINE_MODULES,), - "etb_services": (*_ENGINE_MODULES,), - "uc_reporter_redraw": (uc_relationships, *_ENGINE_MODULES), + "lcfs_consumption": (*_ENGINE_MODULES, energy_pricing, stochastic_assignment), + "etb_vat": (*_ENGINE_MODULES, stochastic_assignment), + "etb_services": (*_ENGINE_MODULES, stochastic_assignment), + "uc_reporter_redraw": (uc_relationships, *_ENGINE_MODULES, stochastic_assignment), # The SPI benefit pass restores from the FRS twin through the support # channel's lineage, re-derives the disability flags and the own-right # rule, draws over the take-up population and maps household weights to @@ -237,8 +255,9 @@ frs_education, frs_take_up, uc_capital_coherence, + stochastic_assignment, ), - "uc_capital_coherence": (uc_relationships,), + "uc_capital_coherence": (uc_relationships, stochastic_assignment), # The Universal Credit redraw reads the award through the adapter, the # One engine materialization of the proxy, the claimant role and carer # status; the frs_spine carrier's codes; the receipt's support counts @@ -256,15 +275,28 @@ uc_relationships, uc_capital_coherence, receipt_counts, + stochastic_assignment, ), # The Move to UC claim draw reuses the UC stage's household-to-benefit-unit # weight mapping and reads the vendored DWP rows (microcosm#1095). - "uc_legacy_closure_take_up": (uc_capital_coherence, ledger_fact_vendoring), + "uc_legacy_closure_take_up": ( + uc_capital_coherence, + ledger_fact_vendoring, + stochastic_assignment, + ), # The Pension Credit redraw reuses the UC stage's household-to-benefit-unit # weight mapping and the adapter's engine materialization. - "pension_credit_take_up": (uc_capital_coherence, *_ENGINE_MODULES), + "pension_credit_take_up": ( + uc_capital_coherence, + *_ENGINE_MODULES, + stochastic_assignment, + ), # The Child Benefit redraw shares the weight mapping and the adapter. - "child_benefit_take_up": (uc_capital_coherence, *_ENGINE_MODULES), + "child_benefit_take_up": ( + uc_capital_coherence, + *_ENGINE_MODULES, + stochastic_assignment, + ), # The Carer's Allowance draw reads the earnings test through the adapter's # engine materialization and counts its receipt's support on the source # lineage (microcosm#1319), and balances its draws through the shared @@ -276,6 +308,12 @@ ), # The stored attendance is one engine materialization (microcosm#1095). "private_school_attendance": (*_ENGINE_MODULES,), + # These four stages draw through the keyed-draw helpers and bind nothing + # else (microcosm#931). + "spi_housing_shell": (stochastic_assignment,), + "uc_deduction_attributes": (stochastic_assignment,), + "salary_sacrifice": (stochastic_assignment,), + "student_loans": (stochastic_assignment,), # The subscriptions walk the CGT bands on its income proxy (with the # proxy's loss relief), read HMRC's ISA tables through their typed loader # and round their receipts' support (microcosm#1415). @@ -284,6 +322,7 @@ hmrc_income, hmrc_isa_statistics, receipt_counts, + stochastic_assignment, ), # The points draws are keyed on identity, read the vendored DWP tables and # round their receipt's support (microcosm#1095, P1). @@ -310,7 +349,12 @@ # The copies' childcare re-draw draws through frs_take_up's childcare # helper, reads each household's clone path, and measures its flags for # the stage-time signal gate by the shared disclosure rules (microcosm#1095). - "childcare_copy_take_up": (frs_take_up, household_lineage, receipt_counts), + "childcare_copy_take_up": ( + frs_take_up, + household_lineage, + receipt_counts, + stochastic_assignment, + ), # The employment shell fits, scores and draws through microcosm.fit's # categorical classifier, held-out scores and keyed donors, reads the # shared occupation codes, and counts its receipt's support by the shared diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index 1372c08a00..177b867513 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -727,21 +727,24 @@ def validate_uk_ladder_rowwise_dataset_tables( _require_unique(person, "person_id", label="person") _require_unique(benunit, "benunit_id", label="benunit") _require_unique(household, "household_id", label="household") + _require_person_links(person, benunit, household) - household_ids = set(household["household_id"]) - missing_households = sorted(set(person["person_household_id"]) - household_ids) - if missing_households: - raise ValueError( - "person.person_household_id contains value(s) absent from household: " - f"{missing_households[:5]}." - ) - benunit_ids = set(benunit["benunit_id"]) - missing_benunits = sorted(set(person["person_benunit_id"]) - benunit_ids) - if missing_benunits: - raise ValueError( - "person.person_benunit_id contains value(s) absent from benunit: " - f"{missing_benunits[:5]}." - ) + +def _require_person_links( + person: pd.DataFrame, benunit: pd.DataFrame, household: pd.DataFrame +) -> None: + """Refuse a person whose household or benefit unit is absent, by count.""" + + for column, table, id_column, label in ( + ("person_household_id", household, "household_id", "household"), + ("person_benunit_id", benunit, "benunit_id", "benunit"), + ): + unlinked = int((~person[column].isin(table[id_column])).sum()) + if unlinked: + raise ValueError( + f"person.{column} names a {label} absent from {label} on " + f"{unlinked} person row(s)." + ) def _assert_clone_link_alignment( @@ -1021,22 +1024,7 @@ def validate_uk_rowwise_dataset_tables( _require_unique(person, "person_id", label="person") _require_unique(benunit, "benunit_id", label="benunit") _require_unique(household, "household_id", label="household") - - household_ids = set(household["household_id"]) - missing_households = sorted(set(person["person_household_id"]) - household_ids) - if missing_households: - raise ValueError( - "person.person_household_id contains value(s) absent from household: " - f"{missing_households[:5]}." - ) - - benunit_ids = set(benunit["benunit_id"]) - missing_benunits = sorted(set(person["person_benunit_id"]) - benunit_ids) - if missing_benunits: - raise ValueError( - "person.person_benunit_id contains value(s) absent from benunit: " - f"{missing_benunits[:5]}." - ) + _require_person_links(person, benunit, household) def read_uk_single_year_weight_metadata( @@ -1281,11 +1269,11 @@ def _require_columns( def _require_unique(frame: pd.DataFrame, column: str, *, label: str) -> None: if frame[column].isna().any(): raise ValueError(f"{label}.{column} contains missing values.") - if frame[column].duplicated().any(): - duplicates = frame.loc[frame[column].duplicated(), column].unique() + repeated = int(frame[column].duplicated().sum()) + if repeated: raise ValueError( - f"{label}.{column} must be unique; duplicate value(s): " - f"{list(map(str, duplicates[:5]))}." + f"{label}.{column} must be unique; {repeated} row(s) repeat an " + "earlier row's value." ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py index 69e8718402..7fe3122e42 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py @@ -362,12 +362,12 @@ def assign_household_geography( clone_frames.append(clone) result = pd.concat(clone_frames, ignore_index=True) - if result["household_id"].duplicated().any(): - duplicates = result.loc[result["household_id"].duplicated(), "household_id"] + repeated = int(result["household_id"].duplicated().sum()) + if repeated: raise ValueError( "remapped 'household_id' values are not unique; id_multiplier is " - "too small for household cloning. Duplicate value(s): " - f"{list(map(str, duplicates.unique()[:5]))}." + f"too small for household cloning: {repeated} row(s) repeat an " + "earlier row's id." ) return RowwiseGeographyAssignment( household=result, @@ -523,12 +523,12 @@ def clone_entity_frame( clones.append(clone) result = pd.concat(clones, ignore_index=True) for column in unique_input_columns: - if result[column].duplicated().any(): - duplicates = result.loc[result[column].duplicated(), column].unique() + repeated = int(result[column].duplicated().sum()) + if repeated: raise ValueError( f"remapped {column!r} values are not unique; id_multiplier " - "is too small for linked entity cloning. Duplicate value(s): " - f"{list(map(str, duplicates[:5]))}." + f"is too small for linked entity cloning: {repeated} row(s) " + "repeat an earlier row's id." ) return result @@ -578,11 +578,11 @@ def _prepare_households( frame = frame.rename(columns={weight_column: "household_weight"}) if frame["household_id"].isna().any(): raise ValueError("household_id contains missing values.") - if frame["household_id"].duplicated().any(): - duplicates = frame.loc[frame["household_id"].duplicated(), "household_id"] + repeated = int(frame["household_id"].duplicated().sum()) + if repeated: raise ValueError( - "household_id must be unique before cloning; duplicate value(s): " - f"{list(map(str, duplicates.unique()[:5]))}." + f"household_id must be unique before cloning; {repeated} row(s) " + "repeat an earlier row's id." ) weights = pd.to_numeric(frame["household_weight"], errors="coerce") if weights.isna().any(): @@ -854,13 +854,18 @@ def _remap_ids( values = pd.to_numeric(pd.Series(ids), errors="raise").astype("int64").to_numpy() if clone_index == 0: return values.copy() - offset = clone_index * id_multiplier - max_value = int(values.max()) if len(values) else 0 - if max_value > 0 and offset > np.iinfo(np.int64).max - max_value: + # Exact: the offset is a Python integer and is compared, never cast, so an + # offset beyond int64 is refused rather than wrapped. The message prints + # counts, never an id or the multiplier. + offset = int(clone_index) * int(id_multiplier) + largest = int(np.iinfo(np.int64).max) + overflowing = ( + len(values) if offset > largest else int((values > largest - offset).sum()) + ) + if overflowing: raise ValueError( - "clone ID remapping would overflow int64: clone_index " - f"{clone_index} x id_multiplier {id_multiplier} + max id " - f"{max_value} exceeds the representable range." + f"clone ID remapping would overflow int64 on {overflowing} of " + f"{len(values)} id(s) at clone index {clone_index}." ) return values + offset diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py index 18f41e0230..2074068da6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py @@ -495,19 +495,90 @@ def fit_tables(run: RowwiseRun) -> dict[str, Any]: } +_INT64_MAX = int(np.iinfo(np.int64).max) + + +def _integer_ids(values: pd.Series, *, label: str) -> np.ndarray: + """An id column as exact int64; a non-integer column is refused, not cast.""" + + if not pd.api.types.is_integer_dtype(values.dtype): + raise TypeError(f"{label} must be an integer column, not {values.dtype}.") + return values.to_numpy(dtype=np.int64) + + def _source_multiplier( weights: pd.DataFrame, spine_weights: pd.DataFrame | None, + recorded: int | None = None, ) -> tuple[int, str]: + """The dense clone multiplier of ``weights``' ids, and where it was read. + + The dense expand offsets clone ``c`` of a spine household by ``c`` times + ten to the digit count of the spine's largest id (the dense line's own + rule, ``id_multiplier_for_values``, until microcosm#931 (c) re-bases it + onto the clone-first spine). A multiplier the dataset recorded (the + expansion artifact's or the row-wise manifest's ``id_multiplier``) is + taken as it is; otherwise it is read off the spine, or off the clone-0 + ids. A Python integer: a K=1 expansion of a spine minted from the copy-id + table records 10**19, which int64 cannot hold. + """ + + if recorded is not None: + if isinstance(recorded, bool) or not isinstance(recorded, int | np.integer): + raise TypeError("a recorded dense id multiplier must be an integer") + if int(recorded) < 1: + raise ValueError("a recorded dense id multiplier must be positive") + return int(recorded), "recorded" if spine_weights is not None: - return ( - id_multiplier_for_values(spine_weights["household_id"].tolist()), - "spine", + spine_ids = _integer_ids( + spine_weights["household_id"], label="spine household_id" ) - originals = weights.loc[weights["clone_index"] == 0, "household_id"] - if originals.empty: + return id_multiplier_for_values(spine_ids), "spine" + ids = _integer_ids(weights["household_id"], label="household_id") + clone_index = _integer_ids(weights["clone_index"], label="clone_index") + originals = ids[clone_index == 0] + if originals.size == 0: raise ValueError("cannot infer source-id multiplier without clone_index == 0") - return 10 ** len(str(int(originals.max()))), "inferred_from_clone_zero_ids" + return id_multiplier_for_values(originals), "inferred_from_clone_zero_ids" + + +def _spine_ids(weights: pd.DataFrame, multiplier: int) -> np.ndarray: + """Each dense row's spine household id: its id less clone_index x multiplier. + + Exact int64, with the multiplier a Python integer. Dense clone ``c`` of + spine household ``s`` carries ``s + c x multiplier``, ``s`` below the + multiplier, so a row that does not decode to such an ``s`` is refused by + count: the multiplier is not the dataset's. At one clone every index is + zero and every id is its spine id, whatever the multiplier, 10**19 + included. The refusals print counts, never an id. + """ + + ids = _integer_ids(weights["household_id"], label="household_id") + clone_index = _integer_ids(weights["clone_index"], label="clone_index") + negative = int((clone_index < 0).sum()) + if negative: + raise ValueError(f"{negative} dense row(s) carry a negative clone_index") + # The largest clone index whose offset int64 holds; zero when the + # multiplier is itself beyond int64. + largest_clone = _INT64_MAX // multiplier + beyond = int((clone_index > largest_clone).sum()) + if beyond: + raise ValueError( + f"{beyond} dense row(s) carry a clone_index whose offset int64 cannot " + "hold at the dense id multiplier; the multiplier is not this dataset's" + ) + if largest_clone: + spine_ids = ids - clone_index * np.int64(multiplier) + else: + spine_ids = ids.copy() + off = int(((spine_ids < 0) | (spine_ids >= multiplier)).sum()) + if off: + raise ValueError( + f"{off} dense row(s) do not sit at their clone_index times the dense " + "id multiplier above a spine id below it; the multiplier is not this " + "dataset's (pass the id_multiplier its expansion recorded)" + ) + return spine_ids def _ratio_summary(ratios: Sequence[float] | np.ndarray) -> dict[str, float]: @@ -528,8 +599,14 @@ def weight_tables( *, spine_weights: pd.DataFrame | None = None, n_clones: int | None = None, + id_multiplier: int | None = None, ) -> dict[str, Any]: - """Summarize shipped weights, source support, and design stretch.""" + """Summarize shipped weights, source support, and design stretch. + + Dense ids decode back to spine ids at the dense clone multiplier: the one + ``id_multiplier`` names (what the dataset's expansion recorded), else the + spine's, else the one the clone-0 ids give (``_source_multiplier``). + """ manifest_weights = _mapping(run.manifest.get("weights")) reference = manifest_weights.get("stretch_reference", "pool_design") @@ -552,6 +629,7 @@ def weight_tables( weight_frame = run.selection[ ["household_id", "refit_weight", "clone_index"] ].rename(columns={"refit_weight": "household_weight"}) + spine_ids = None if weight_frame is not None: values = weight_frame["household_weight"].to_numpy(dtype=np.float64) result["summary"] = uk_weight_summary(values) @@ -565,11 +643,11 @@ def weight_tables( else: # Without the column only the geographic clones fold back: the id # arithmetic recovers each spine household, copies apart. - multiplier, basis = _source_multiplier(weight_frame, spine_weights) - base_ids = ( - weight_frame["household_id"].to_numpy(dtype=np.int64) - - weight_frame["clone_index"].to_numpy(dtype=np.int64) * multiplier + multiplier, basis = _source_multiplier( + weight_frame, spine_weights, id_multiplier ) + spine_ids = _spine_ids(weight_frame, multiplier) + base_ids = spine_ids result["distinct_source_households"] = int(np.unique(base_ids).size) result["source_id_multiplier"] = multiplier result["source_id_multiplier_basis"] = basis @@ -600,14 +678,16 @@ def weight_tables( clones = n_clones or int( _mapping(run.manifest.get("parameters")).get("n_clones") ) - multiplier = int(result["source_id_multiplier"]) + if spine_ids is None: + # A file that keeps its true sources still decodes its dense ids to + # the spine households the spine weights are keyed on. + multiplier, _ = _source_multiplier( + weight_frame, spine_weights, id_multiplier + ) + spine_ids = _spine_ids(weight_frame, multiplier) source_map = spine_weights.set_index("household_id")["household_weight"] - base_ids = ( - weight_frame["household_id"].to_numpy(dtype=np.int64) - - weight_frame["clone_index"].to_numpy(dtype=np.int64) * multiplier - ) try: - design = source_map.loc[base_ids].to_numpy(dtype=np.float64) / clones + design = source_map.loc[spine_ids].to_numpy(dtype=np.float64) / clones except KeyError as exc: raise ValueError( "spine weights do not cover every source household" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 3b40189de6..ebd729a0bb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -568,8 +568,8 @@ def __call__(self, frame: Frame) -> Frame: missing_ids = sorted(set(source_leaves.index) - set(person["person_id"])) if missing_ids and not self.sampled_rung: raise ValueError( - "Raw FRS retained leaves contain person identity value(s) absent " - f"from the raw spine: {missing_ids[:5]}." + f"Raw FRS retained leaves contain {len(missing_ids)} person " + "identity value(s) absent from the raw spine." ) if missing_ids: # A rung sample deliberately drops most source people; restrict diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py index 7b21e65380..10d01a8ee4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py @@ -664,11 +664,11 @@ def fill_support_channel_from_source( raise ValueError(f"donor is missing column(s): {missing_donor}.") if donor[donor_id_column].isna().any(): raise ValueError(f"donor.{donor_id_column} contains missing values.") - if donor[donor_id_column].duplicated().any(): - duplicates = donor.loc[donor[donor_id_column].duplicated(), donor_id_column] + repeated = int(donor[donor_id_column].duplicated().sum()) + if repeated: raise ValueError( - f"donor.{donor_id_column} must be unique; duplicate value(s): " - f"{list(map(str, duplicates.unique()[:5]))}." + f"donor.{donor_id_column} must be unique; {repeated} row(s) repeat " + "an earlier row's id." ) channel_column = support_channel_column(entity) @@ -690,11 +690,11 @@ def fill_support_channel_from_source( donor_indexed = donor.set_index(donor_id_column, drop=False) source_ids = out.loc[mask, source_id_column] - missing_ids = source_ids[~source_ids.isin(donor_indexed.index)].unique() - if len(missing_ids): + unmatched = int((~source_ids.isin(donor_indexed.index)).sum()) + if unmatched: raise ValueError( "donor is missing source ID value(s) required by the support " - f"channel: {list(map(str, missing_ids[:5]))}." + f"channel: {unmatched} channel row(s) name a source it lacks." ) aligned = donor_indexed.loc[source_ids.to_numpy()] @@ -824,11 +824,10 @@ def _validate_selected_household_ids( if len(set(selected)) != len(selected): raise ValueError("selected_household_ids must be unique.") available = set(household["household_id"]) - missing = [value for value in selected if value not in available] + missing = sum(1 for value in selected if value not in available) if missing: raise ValueError( - "selected_household_ids contains value(s) absent from household: " - f"{list(map(str, missing[:5]))}." + f"selected_household_ids contains {missing} value(s) absent from household." ) selected_set = set(selected) return tuple( @@ -1191,21 +1190,16 @@ def _validate_uk_support_inputs( if not np.isfinite(weights).all() or (weights < 0).any(): raise ValueError("household.household_weight must be finite and non-negative.") - household_ids = set(household["household_id"]) - missing_households = sorted(set(person["person_household_id"]) - household_ids) - if missing_households: - raise ValueError( - "person.person_household_id contains value(s) absent from household: " - f"{missing_households[:5]}." - ) - - benunit_ids = set(benunit["benunit_id"]) - missing_benunits = sorted(set(person["person_benunit_id"]) - benunit_ids) - if missing_benunits: - raise ValueError( - "person.person_benunit_id contains value(s) absent from benunit: " - f"{missing_benunits[:5]}." - ) + for column, table, id_column, label in ( + ("person_household_id", household, "household_id", "household"), + ("person_benunit_id", benunit, "benunit_id", "benunit"), + ): + unlinked = int((~person[column].isin(table[id_column])).sum()) + if unlinked: + raise ValueError( + f"person.{column} names a {label} absent from {label} on " + f"{unlinked} person row(s)." + ) def _validate_uk_support_outputs( @@ -1265,11 +1259,11 @@ def _require_columns( def _require_unique(frame: pd.DataFrame, column: str, *, label: str) -> None: if frame[column].isna().any(): raise ValueError(f"{label}.{column} contains missing values.") - if frame[column].duplicated().any(): - duplicates = frame.loc[frame[column].duplicated(), column].unique() + repeated = int(frame[column].duplicated().sum()) + if repeated: raise ValueError( - f"{label}.{column} must be unique; duplicate value(s): " - f"{list(map(str, duplicates[:5]))}." + f"{label}.{column} must be unique; {repeated} row(s) repeat an " + "earlier row's value." ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 0a7377817d..d13702fc2b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -269,10 +269,12 @@ def test_income_band_refuses_a_household_without_an_adult_or_a_missing_component person, _, household, _ = support_tables( weights=[1.0], wealth=[0.0], incomes=[(0.0,)], ages=[(12,)] ) - with pytest.raises(ValueError, match="adult carrier"): + with pytest.raises(ValueError, match="adult carrier") as caught: cgt_support_income_band( person, household_ids=household["household_id"], parameters=PARAMETERS ) + # The refusal counts the households and names none (microcosm#931). + assert str(caught.value).endswith("1 household(s) have none.") person, _, household, _ = support_tables(weights=[1.0], wealth=[0.0], incomes=[1.0]) with pytest.raises(ValueError, match="components missing"): cgt_support_income_band( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py index 7aac59350c..ae65724301 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -10,7 +10,6 @@ from __future__ import annotations import ast -import inspect import json import re from pathlib import Path @@ -71,6 +70,7 @@ copy_households, id_frame, ) +from test_support.paths import paths_for _FIELD_NAMES = ( "vintage", @@ -604,6 +604,33 @@ def test_the_executor_refuses_new_rows_without_an_incumbent_parent_by_count() -> ) as caught: graph_kernels._source_lineage(before, after, "household", id_offset=None) assert str(stray) not in str(caught.value) + # The SPI support stages name each new row's parent in household_source_id: + # the copy names an incumbent, the stray a household the frame never held. + orphan_source = 10**11 + 77_777 + household = after_tables.household.copy() + incumbent = household["household_id"].isin(before_tables.household["household_id"]) + sources = household["household_id"].where( + incumbent, int(before_tables.household["household_id"].iloc[0]) + ) + sources[household["household_id"] == stray] = orphan_source + household["household_source_id"] = sources.astype("int64") + sourced = id_frame( + IdTables( + person=after_tables.person, + benunit=after_tables.benunit, + household=household, + ) + ) + with pytest.raises( + ValueError, + match=( + r"^UK stage produced 1 new 'household' row\(s\) whose " + r"'household_source_id' is not an incumbent id\.$" + ), + ) as caught: + graph_kernels._source_lineage(before, sourced, "household", id_offset=None) + assert str(stray) not in str(caught.value) + assert str(orphan_source) not in str(caught.value) def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: @@ -622,34 +649,117 @@ def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: # --- no UK spine module keeps the retired rule ------------------------------------ - -@pytest.mark.parametrize( - "module", - [ - "spi_support", - "spi_band_donors", - "high_value_home_support", - "cgt_support", - "cgt_structure", - "cgt_residential_split", - "graph_kernels", - ], +#: The rule the table replaced: ten to the digit count of the largest id seen. +_RETIRED_RULE = "id_multiplier_for_values" +#: The modules that keep the dense line's own clone multiplier, the retired +#: rule over the spine's ids, until #931 (c) re-bases the dense line onto the +#: clone-first spine, each with what it uses it for. Nothing else may name it. +_DENSE_RULE_MODULES = { + "packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py": ( + "defines the dense clone multiplier and clones linked frames at it" + ), + "packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py": ( + "the dense expand and the crosswalk clone route offset dense clones by it" + ), + "packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py": ( + "re-exports the dense helper" + ), + "packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py": ( + "decodes dense clone ids back to spine ids" + ), + "tools/build_uk_rowwise_dataset.py": ( + "the row-wise tool's dry-run plans size the dense clone's ids by it" + ), +} +#: Every module that mints, bounds or decodes spine copy ids; none may also +#: name the retired rule. Each reads the copy-id table, save the band +#: donors, which mint through the support channel's clone helper. +_COPY_ID_MODULES = frozenset( + { + *( + f"packages/microcosm-build/src/microcosm/build/uk_runtime/{module}.py" + for module in ( + "cgt_residential_split", + "cgt_structure", + "cgt_support", + "frs_spine", + "graph_kernels", + "high_value_home_support", + "spi_band_donors", + "spi_support", + ) + ), + "tools/verify_uk_identity_stability.py", + } ) -def test_no_uk_spine_stage_module_imports_the_retired_id_rule(module) -> None: - import importlib +_MINTS_THROUGH_SPI_SUPPORT = frozenset( + {"packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py"} +) + + +def _uk_python_sources() -> dict[str, str]: + """The UK runtime's and the UK tools' sources by repository path.""" + + paths = paths_for("microcosm-build") + root = paths.repository + runtime = paths.package / "src" / "microcosm" / "build" / "uk_runtime" + uk_tool = re.compile(r"(?:^|_)uk(?:_|$)") + sources = ( + *runtime.rglob("*.py"), + *(path for path in (root / "tools").glob("*.py") if uk_tool.search(path.stem)), + ) + return { + path.relative_to(root).as_posix(): path.read_text(encoding="utf-8") + for path in sources + } + + +def _source_names(source: str) -> set[str]: + """Every name a source imports, reads or reaches by attribute.""" - source = Path( - inspect.getfile(importlib.import_module(f"microcosm.build.uk_runtime.{module}")) - ).read_text(encoding="utf-8") names = set() for node in ast.walk(ast.parse(source)): if isinstance(node, ast.ImportFrom): names.update(alias.name for alias in node.names) + if node.module: + names.add(node.module.rsplit(".", 1)[-1]) + elif isinstance(node, ast.Import): + names.update(alias.name.rsplit(".", 1)[-1] for alias in node.names) elif isinstance(node, ast.Attribute): names.add(node.attr) elif isinstance(node, ast.Name): names.add(node.id) - assert "id_multiplier_for_values" not in names + return names + + +def test_only_the_dense_line_names_the_retired_id_rule() -> None: + """The retired rule survives only where the dense line keeps its own + multiplier until #931 (c); a stale entry fails as well as a new user.""" + + names = { + path: _source_names(source) for path, source in _uk_python_sources().items() + } + assert {path for path, found in names.items() if _RETIRED_RULE in found} == set( + _DENSE_RULE_MODULES + ) + + +def test_no_module_that_mints_or_decodes_copy_ids_names_the_retired_rule() -> None: + """Each module that mints, bounds or decodes spine copy ids reads the + copy-id table, the identity tool among them, and none names the rule the + table replaced; the readers of the table are exactly these modules.""" + + names = { + path: _source_names(source) for path, source in _uk_python_sources().items() + } + readers = {path for path, found in names.items() if "copy_id_table" in found} + readers.discard( + "packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py" + ) + assert readers == _COPY_ID_MODULES - _MINTS_THROUGH_SPI_SUPPORT + for path in sorted(_COPY_ID_MODULES): + assert _RETIRED_RULE not in names[path], path + assert not _COPY_ID_MODULES & set(_DENSE_RULE_MODULES) # --- the dense line --------------------------------------------------------------- diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index f763d29bd4..1fb058a125 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -420,6 +420,90 @@ def changed_helper_bytes(path: Path) -> bytes: assert registry.implementation_hash(ref) == before[ref], ref +#: Stages whose module or bound helpers import the keyed-draw helpers +#: (``microcosm.build.stochastic_assignment``) but need not bind them, each with +#: its reason. None today: every stage that imports them draws through them. +_STOCHASTIC_ASSIGNMENT_UNBOUND: dict[str, str] = {} + + +def _stages_drawing_through_stochastic_assignment() -> set[str]: + from microcosm.build.uk_runtime import graph_kernels + + return { + stage + for stage in graph_kernels._STAGE_MODULES + if any( + "stochastic_assignment" in _imported_module_names(module) + for module in ( + graph_kernels._stage_module(stage), + *graph_kernels._STAGE_HELPER_MODULES.get(stage, ()), + ) + ) + } + + +def test_every_stage_that_draws_through_stochastic_assignment_binds_it() -> None: + """``source_hash`` does not follow imports, so every stage whose module or + bound helpers import the keyed-draw helpers binds them, or an edit to how a + uniform is keyed would replay the stage's stale output (microcosm#931), + save the stages ``_STOCHASTIC_ASSIGNMENT_UNBOUND`` names with a reason.""" + from microcosm.build import stochastic_assignment + from microcosm.build.uk_runtime import graph_kernels + + drawing = _stages_drawing_through_stochastic_assignment() + binding = { + stage + for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() + if stochastic_assignment in helpers + } + assert set(_STOCHASTIC_ASSIGNMENT_UNBOUND) <= drawing + assert binding == drawing - set(_STOCHASTIC_ASSIGNMENT_UNBOUND) + + +def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( + monkeypatch, +): + from microcosm.build import stochastic_assignment + from microcosm.build.uk_runtime import graph_kernels + + graph = uk_spine_graph() + registry = uk_registry(graph=graph) + stages = sorted( + stage + for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() + if stochastic_assignment in helpers + ) + # The root stage's node is the CREATE kernel. + affected = [ + graph.node("create_uk_frs" if stage == "frs_spine" else stage).kernel + for stage in stages + ] + # Three stages that draw through nothing of it. + control_stages = ( + "frs_council_tax", + "cgt_support_split", + "hmrc_cgt_asset_type_spine", + ) + assert not set(control_stages) & _stages_drawing_through_stochastic_assignment() + controls = [graph.node(stage).kernel for stage in control_stages] + before = {ref: registry.implementation_hash(ref) for ref in affected + controls} + helper_path = Path(stochastic_assignment.__file__).resolve() + original = Path.read_bytes + + def changed_helper_bytes(path: Path) -> bytes: + content = original(path) + if path.resolve() == helper_path: + return content + b"\n# A helper-only source change.\n" + return content + + monkeypatch.setattr(Path, "read_bytes", changed_helper_bytes) + assert len(affected) == len(_stages_drawing_through_stochastic_assignment()) + for ref in affected: + assert registry.implementation_hash(ref) != before[ref], ref + for ref in controls: + assert registry.implementation_hash(ref) == before[ref], ref + + def test_isa_statistics_source_changes_invalidate_the_stages_that_read_them( monkeypatch, ): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 3154344f66..aac803ed75 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -19,6 +19,7 @@ import pytest from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.age_tail import ( UK_AGE_TOP_CODE, disaggregate_uk_age_top_code, @@ -736,7 +737,8 @@ def test_e5_receipt_covers_the_spi_rows_the_stage_imputed(self) -> None: # walks by wealth descending then id ascending: id 2 (130) then id 3 (121) # cross 240, so both split into three copies; band 37,700 takes id 6 (61) at # once, two copies. Ids 1, 4 and 5 stay whole; id 5 is the heaviest unselected -# incumbent and takes the exact-total correction. Pre-split multiplier 10. +# incumbent and takes the exact-total correction. Copy k of root r carries +# r + k x 10**13, the copy-id table's cgt_support_split field (microcosm#931). _SPLIT_WEIGHTS = [70.0, 130.0, 121.0, 30.0, 90.0, 61.0] _SPLIT_WEALTH = [100.0, 500.0, 500.0, 200.0, 10.0, 20.0] @@ -781,20 +783,25 @@ def _rebuild(frame, *, household=None, household_weights=None, mass_log=None): def _hand_split_frame(*, cloned: bool = True, anchored: bool = False): """Three source households, the first split into three at a third each. - Pre-split ids 1..3 (persons and benunits alike, multiplier 10) at source - weights 90 / 20 / 40: root 1 and its copies 11 and 21 carry 30 each, - flagged and counted as the stage leaves them. With ``cloned`` the clone - layer follows (multiplier 100) at equal halves, or with ``anchored`` at - an uneven pair split whose pair sums are still the pre-clone weights, as - the #970 anchor leaves them. + Pre-split ids 1..3 (persons and benunits alike) at source weights 90 / 20 + / 40: root 1 and its copies 1 + 10**13 and 1 + 2 x 10**13, minted in the + copy-id table's ``cgt_support_split`` field as the stage mints them, + carry 30 each, flagged and counted as the stage leaves them. With + ``cloned`` the clone layer follows in the ``cgt_incidence_clone`` field + (10**17) at equal halves, or with ``anchored`` at an uneven pair split + whose pair sums are still the pre-clone weights, as the #970 anchor + leaves them. """ - ids = np.array([1, 2, 3, 11, 21], dtype="int64") + ids = np.r_[ + np.array([1, 2, 3], dtype="int64"), + copy_id_table.mint([1, 1], "cgt_support_split", [1, 2]), + ] flags = [False, False, False, True, True] copies = [3, 1, 1, 3, 3] pre_clone = np.array([30.0, 20.0, 40.0, 30.0, 30.0]) if cloned: - ids = np.r_[ids, ids + 100] + ids = np.r_[ids, copy_id_table.mint(ids, "cgt_incidence_clone", 1)] flags = flags + flags copies = copies + copies if anchored: @@ -886,7 +893,8 @@ def test_a_copy_without_a_root_fails_closed(self) -> None: tool = _load_tool() frame = _hand_split_frame(cloned=False) household = frame.table("household").copy() - # Flagging id 2 as a copy gives it index 2 // 10 = 0: no root. + # Flagging id 2 as a copy gives it ordinal 0 in the cgt_support_split + # field: no root. household.loc[household["household_id"] == 2, HOUSEHOLD_IS_CGT_SUPPORT_COPY] = ( True ) @@ -902,9 +910,7 @@ def test_the_stored_copy_index_is_read_and_held_to_the_id_scheme(self) -> None: tool = _load_tool() _, frame = _split_then_cloned() household = frame.table("household") - lineage = tool._support_copy_lineage( - frame.table("person"), frame.table("benunit"), household - ) + lineage = tool._support_copy_lineage(household) stored = household[CGT_SUPPORT_COPY_INDEX_COLUMN].to_numpy() np.testing.assert_array_equal( stored[lineage.copy_positions], lineage.copy_index @@ -955,7 +961,8 @@ def test_a_frame_the_stage_built_receipts_green(self) -> None: assert receipt["copies_recomputed"] == 5 assert receipt["households_selected"] == 3 assert receipt["pre_split_households"] == 6 - assert receipt["id_multiplier"] == 10 + # The receipt records the split field's declared radix. + assert receipt["id_multiplier"] == copy_id_table.radix("cgt_support_split") # 2 x 1.25 x (60 + 10) published in the two bands. assert receipt["support_mass"] == 175.0 assert receipt["mass"]["old_total"] == receipt["mass"]["new_total"] @@ -975,8 +982,9 @@ def test_a_moved_copy_weight_is_reported(self) -> None: weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() # A unit of mass moves between root 2's two copies: the family total # holds, so the selection stands, but two members leave w / n. - weights[ids == 12] += 1.0 - weights[ids == 22] -= 1.0 + first, second = copy_id_table.mint([2, 2], "cgt_support_split", [1, 2]) + weights[ids == first] += 1.0 + weights[ids == second] -= 1.0 _, problems = self._receipt(_rebuild(frame, household_weights=weights)) assert problems == {"support_split_family_weights": 2} @@ -1026,3 +1034,190 @@ def test_an_artifact_without_the_split_layer_is_refused(self) -> None: ) with pytest.raises(ValueError, match="support-split layer is absent"): self._receipt(frame) + + +# --- copy ids at the copy-id table's declared radices (microcosm#931) ---------- +# +# The decoders read every copy id at its field's declared radix, never at a +# power of ten above the frame's largest id, and the tool refuses a spine whose +# copy ids follow the rule the table replaced. Every frame here is synthetic. + + +def _lineage_household(household: pd.DataFrame) -> pd.DataFrame: + """A copy chain's path columns as the spine's own lineage columns.""" + + from microcosm.build.uk_runtime.cgt_residential_split import ( + CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, + HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, + ) + from microcosm.build.uk_runtime.cgt_support import CGT_SUPPORT_COPY_INDEX_COLUMN + from microcosm.build.uk_runtime.household_lineage import ( + HIGH_VALUE_SUPPORT_COPY_COLUMN, + HIGH_VALUE_SUPPORT_INDEX_COLUMN, + ) + from test_support.microcosm_build.uk_clone_first_frames import COPY_PATH_COLUMNS + + lineage = household.drop(columns=list(COPY_PATH_COLUMNS)) + lineage["household_support_clone_index"] = household["spi"] + lineage["household_support_channel"] = np.where(household["spi"] > 0, "spi", "frs") + lineage[HIGH_VALUE_SUPPORT_INDEX_COLUMN] = household["hv"] + lineage[HIGH_VALUE_SUPPORT_COPY_COLUMN] = household["hv"] > 0 + lineage[CGT_SUPPORT_COPY_INDEX_COLUMN] = household["cgt"] + lineage[HOUSEHOLD_IS_CGT_SUPPORT_COPY] = household["cgt"] > 0 + lineage[HOUSEHOLD_IS_CGT_CLONE] = household["twin"] > 0 + lineage[CGT_RESIDENTIAL_CLONE_INDEX_COLUMN] = household["arm"] + lineage[HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE] = household["arm"] > 0 + return lineage + + +def _chain(rule: str): + """The six copy layers on synthetic sernums, minted by ``rule``.""" + + from test_support.microcosm_build.uk_clone_first_frames import ( + base_id_tables, + copy_chain, + ) + + return copy_chain(base_id_tables(), rule=rule) + + +class TestCopyIdsAtTheDeclaredTable: + def test_a_spine_the_table_minted_follows_it(self) -> None: + tool = _load_tool() + household = _lineage_household(_chain("table").household) + assert tool._copy_id_rows_off_the_declared_table(household) == 0 + tool._require_copy_ids_on_the_declared_table(household) + + def test_a_spine_minted_by_the_retired_rule_is_refused_by_count(self) -> None: + tool = _load_tool() + household = _lineage_household(_chain("retired").household) + off = tool._copy_id_rows_off_the_declared_table(household) + # Every copy row: the retired rule put each layer's copies a power of + # ten above the largest id the layer saw, never in the layer's field. + copies = int((household["household_id"] >= 10**5).sum()) + assert off == copies > 0 + with pytest.raises(ValueError) as caught: + tool._require_copy_ids_on_the_declared_table(household) + assert str(caught.value) == f"{off} {tool._OFF_TABLE_REFUSAL}" + assert "commit that built it" in str(caught.value) + + def test_a_copy_whose_lineage_disagrees_with_its_id_is_refused(self) -> None: + tool = _load_tool() + household = _lineage_household(_chain("table").household) + # One twin loses its flag; its id still holds the twin field's ordinal. + twin = int(np.flatnonzero(household[HOUSEHOLD_IS_CGT_CLONE].to_numpy())[0]) + household.loc[twin, HOUSEHOLD_IS_CGT_CLONE] = False + assert tool._copy_id_rows_off_the_declared_table(household) == 1 + + def test_the_tool_refuses_a_spine_built_before_the_table( + self, monkeypatch, tmp_path + ) -> None: + import sys + + from test_support.microcosm_build.uk_clone_first_frames import ( + IdTables, + id_frame, + ) + + tool = _load_tool() + tables = _chain("retired") + frame = id_frame( + IdTables( + person=tables.person, + benunit=tables.benunit, + household=_lineage_household(tables.household), + ) + ) + off = tool._copy_id_rows_off_the_declared_table(frame.table("household")) + monkeypatch.setattr(tool, "load_uk_national_frame", lambda path: (frame, {})) + output = tmp_path / "receipt.json" + monkeypatch.setattr( + sys, + "argv", + [ + "verify_uk_identity_stability.py", + "--input-h5", + str(tmp_path / "spine.h5"), + "--output", + str(output), + "--check", + "e7", + ], + ) + with pytest.raises(ValueError) as caught: + tool.main() + assert str(caught.value) == f"{off} {tool._OFF_TABLE_REFUSAL}" + assert not output.exists() + + def test_the_arms_decode_at_the_declared_radix_above_2_53(self) -> None: + tool = _load_tool() + tables = _chain("table") + path = ["source_household_id", "spi", "hv", "cgt", "twin"] + lineage = tool._residential_arm_lineage(_lineage_household(tables.household)) + household = tables.household + arms = household.iloc[lineage.arm_positions] + assert len(arms) > 0 + assert int(arms["household_id"].min()) > 2**53 + assert lineage.radix == copy_id_table.radix("cgt_residential_split") + np.testing.assert_array_equal(lineage.arm_index, arms["arm"].to_numpy()) + # Each arm's source is the row with its copy path and no arm, found by + # lineage, never by id arithmetic. + sources = household.loc[household["arm"] == 0, path] + position_by_path = { + tuple(row): position + for position, row in zip( + sources.index, sources.itertuples(index=False), strict=True + ) + } + expected = [ + position_by_path[tuple(row)] for row in arms[path].itertuples(index=False) + ] + np.testing.assert_array_equal(lineage.source_positions, expected) + + def test_a_stored_arm_index_off_the_id_and_an_orphan_arm_are_refused( + self, + ) -> None: + from microcosm.build.uk_runtime.cgt_residential_split import ( + CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, + ) + + tool = _load_tool() + household = _lineage_household(_chain("table").household) + arm = int(np.flatnonzero(household[CGT_RESIDENTIAL_CLONE_INDEX_COLUMN] > 0)[0]) + tampered = household.copy() + tampered.loc[arm, CGT_RESIDENTIAL_CLONE_INDEX_COLUMN] = 7 + with pytest.raises(ValueError, match="disagrees with the id scheme") as caught: + tool._residential_arm_lineage(tampered) + assert str(household["household_id"].iloc[arm]) not in str(caught.value) + orphan = household.copy() + source = int(orphan["household_id"].iloc[arm]) - int( + orphan[CGT_RESIDENTIAL_CLONE_INDEX_COLUMN].iloc[arm] + ) * copy_id_table.radix("cgt_residential_split") + orphan = orphan.loc[orphan["household_id"] != source].reset_index(drop=True) + with pytest.raises(ValueError, match="without a source household") as caught: + tool._residential_arm_lineage(orphan) + assert str(source) not in str(caught.value) + + def test_a_support_copy_decodes_exactly_above_2_53(self) -> None: + from microcosm.build.uk_runtime.cgt_support import ( + CGT_SUPPORT_COPY_INDEX_COLUMN, + ) + + tool = _load_tool() + # The field's largest ordinal: 1 + 9,999 x 10**13 has no float64 twin, + # so a float read would name a root the frame does not hold. + copy = int(copy_id_table.mint(1, "cgt_support_split", 9_999)) + assert copy > 2**53 and float(copy) != copy + household = pd.DataFrame( + { + "household_id": np.array([1, 2, copy], dtype=np.int64), + HOUSEHOLD_IS_CGT_SUPPORT_COPY: [False, False, True], + CGT_SUPPORT_COPY_INDEX_COLUMN: np.array([0, 0, 9_999], dtype=np.int64), + } + ) + lineage = tool._support_copy_lineage(household) + assert lineage.copy_index.tolist() == [9_999] + assert lineage.root_positions.tolist() == [0] + floated = household.assign(household_id=household["household_id"].astype(float)) + with pytest.raises(TypeError, match="integer household ids"): + tool._support_copy_lineage(floated) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dataset.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dataset.py index dc3f49c20d..606b484763 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dataset.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dataset.py @@ -214,8 +214,25 @@ def test_validate_uk_rowwise_dataset_tables_rejects_broken_household_link() -> N person = result.person.copy() person.loc[0, "person_household_id"] = 999 - with pytest.raises(ValueError, match="person_household_id"): + with pytest.raises(ValueError, match="person_household_id") as caught: validate_uk_rowwise_dataset_tables(person, result.benunit, result.household) + # The refusals count rows and name no id (microcosm#931). + assert str(caught.value) == ( + "person.person_household_id names a household absent from household on " + "1 person row(s)." + ) + person = result.person.copy() + person.loc[0, "person_benunit_id"] = 987_654 + with pytest.raises(ValueError, match="person_benunit_id") as caught: + validate_uk_rowwise_dataset_tables(person, result.benunit, result.household) + assert "987654" not in str(caught.value) + household = result.household.copy() + household.loc[1, "household_id"] = int(household["household_id"].iloc[0]) + with pytest.raises(ValueError, match="must be unique") as caught: + validate_uk_rowwise_dataset_tables(result.person, result.benunit, household) + assert str(caught.value) == ( + "household.household_id must be unique; 1 row(s) repeat an earlier row's value." + ) def test_clone_uk_dataset_h5_roundtrip(tmp_path) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_geography.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_geography.py index acb7d70ff3..a63c9c8f78 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_geography.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_geography.py @@ -374,6 +374,56 @@ def test_clone_entity_frame_rejects_too_small_multiplier() -> None: ) +def test_clone_refusals_print_counts_and_no_id() -> None: + """The clone helpers' refusals print counts, never an id or the multiplier + (microcosm#931); every id here is synthetic.""" + + households = pd.DataFrame( + { + "household_id": [700_001, 700_002], + "household_weight": [10.0, 20.0], + "region": ["LONDON", "LONDON"], + } + ) + with pytest.raises(ValueError, match="household_id.*id_multiplier") as caught: + assign_household_geography( + households, crosswalk_rows(), n_clones=2, id_multiplier=1 + ) + assert str(caught.value).endswith( + "too small for household cloning: 1 row(s) repeat an earlier row's id." + ) + assert "700002" not in str(caught.value) + + repeated = households.assign(household_id=[700_001, 700_001]) + with pytest.raises(ValueError, match="unique before cloning") as caught: + assign_household_geography(repeated, crosswalk_rows(), n_clones=1) + assert "1 row(s) repeat" in str(caught.value) + assert "700001" not in str(caught.value) + + people = pd.DataFrame({"person_id": [700_001_001, 700_002_001]}) + with pytest.raises(ValueError, match="id_multiplier") as caught: + clone_entity_frame( + people, id_columns=["person_id"], n_clones=2, id_multiplier=1_000 + ) + assert "1 row(s) repeat" in str(caught.value) + assert "700002001" not in str(caught.value) + + largest = np.iinfo(np.int64).max + near = pd.DataFrame({"person_id": [5, largest - 10]}) + for multiplier in (100, 10**19): + with pytest.raises(ValueError, match="overflow int64") as caught: + clone_entity_frame( + near, id_columns=["person_id"], n_clones=2, id_multiplier=multiplier + ) + overflowing = 1 if multiplier == 100 else 2 + assert str(caught.value) == ( + f"clone ID remapping would overflow int64 on {overflowing} of 2 id(s) " + "at clone index 1." + ) + assert str(largest - 10) not in str(caught.value) + assert str(multiplier) not in str(caught.value) + + def test_id_multiplier_can_cover_all_linked_entity_ids() -> None: households = pd.DataFrame({"household_id": [101, 102]}) people = pd.DataFrame( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_size_evaluation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_size_evaluation.py index fe851d874b..cef43e176e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_size_evaluation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_size_evaluation.py @@ -5,6 +5,7 @@ import hashlib import json import re +from dataclasses import replace from pathlib import Path import numpy as np @@ -580,6 +581,173 @@ def test_weight_tables_size_and_dense_spine_paths(tmp_path: Path) -> None: weight_tables(dense, spine_weights=spine) +# --- dense ids back to spine ids, exactly (microcosm#931) ----------------------- +# +# The dense line keeps its own clone multiplier, ten to the digit count of the +# spine's largest id, until #931 (c). A spine minted from the copy-id table +# reaches 7.1e18, so a one-clone expansion of it records 10**19, which int64 +# cannot hold; every id below is synthetic. + + +def _table_minted_spine_ids() -> np.ndarray: + """Two FRS households, an SPI copy and a residential arm of a twin.""" + + from microcosm.build.uk_runtime import copy_id_table + + twin = copy_id_table.mint([2], "cgt_incidence_clone", 1) + return np.concatenate( + [ + np.array([1, 2], dtype=np.int64), + copy_id_table.mint([1], "spi_support_channel", 1), + copy_id_table.mint(twin, "cgt_residential_split", 7), + ] + ) + + +def _with_households(path: Path, household: pd.DataFrame) -> Path: + """The run at ``path`` with its local file's household table replaced.""" + + with pd.HDFStore(path / "microcosm_uk_2025_local.h5", mode="w") as store: + store.put("household", household, format="table") + return path + + +def test_one_dense_clone_of_a_table_minted_spine_decodes_exactly( + tmp_path: Path, +) -> None: + ids = _table_minted_spine_ids() + assert int(ids.max()) > 2**62 + weights = np.array([10.0, 20.0, 30.0, 40.0]) + run = load_run( + _with_households( + _write_run_dir(tmp_path, "dense", size_run=False), + pd.DataFrame( + { + "household_id": ids, + "household_weight": weights, + "clone_index": np.zeros(len(ids), dtype=np.int64), + } + ), + ), + label="dense", + ) + assert run.weights["household_id"].tolist() == ids.tolist() + spine = pd.DataFrame({"household_id": ids, "household_weight": weights}) + by_spine = weight_tables(run, spine_weights=spine, n_clones=1) + inferred = weight_tables(run) + recorded = weight_tables( + run, + spine_weights=spine, + n_clones=1, + # As the expansion artifact records it: canonical JSON keeps it exact. + id_multiplier=json.loads(json.dumps({"id_multiplier": 10**19}))[ + "id_multiplier" + ], + ) + for result, basis in ( + (by_spine, "spine"), + (inferred, "inferred_from_clone_zero_ids"), + (recorded, "recorded"), + ): + assert result["source_id_multiplier"] == 10**19 + assert result["source_id_multiplier_basis"] == basis + assert result["distinct_source_households"] == len(ids) + assert json.loads(json.dumps(by_spine))["source_id_multiplier"] == 10**19 + assert by_spine["stretch"]["vs_pool_design"]["max"] == pytest.approx(1.0) + + +def test_an_older_dense_run_decodes_at_the_multiplier_it_recorded( + tmp_path: Path, +) -> None: + # A spine minted by the retired rule: a copy one power of ten above the + # largest id it saw (11 = 1 + 10), and three dense clones at 100. + spine_ids = np.array([1, 2, 3, 11], dtype=np.int64) + clones = np.repeat(np.arange(3, dtype=np.int64), len(spine_ids)) + dense_ids = np.tile(spine_ids, 3) + clones * 100 + run = load_run( + _with_households( + _write_run_dir(tmp_path, "dense", size_run=False), + pd.DataFrame( + { + "household_id": dense_ids, + "household_weight": np.full(len(dense_ids), 5.0), + "clone_index": clones, + } + ), + ), + label="dense", + ) + expansion = json.loads(json.dumps({"n_clones": 3, "id_multiplier": 100})) + recorded = weight_tables(run, id_multiplier=expansion["id_multiplier"]) + assert recorded["source_id_multiplier"] == 100 + assert recorded["source_id_multiplier_basis"] == "recorded" + assert recorded["distinct_source_households"] == len(spine_ids) + assert weight_tables(run)["source_id_multiplier"] == 100 + # A multiplier that is not the dataset's is refused by count, naming no + # id: at 10 the copy 11 and every clone above zero fall off it. + with pytest.raises(ValueError, match=r"^9 dense row\(s\) do not sit") as caught: + weight_tables(run, id_multiplier=10) + assert not any( + str(value) in str(caught.value) for value in dense_ids if value >= 10 + ) + with pytest.raises(TypeError, match="must be an integer"): + weight_tables(run, id_multiplier=100.0) + + +def test_a_clone_offset_int64_cannot_hold_is_refused_by_count( + tmp_path: Path, +) -> None: + ids = _table_minted_spine_ids() + run = load_run( + _with_households( + _write_run_dir(tmp_path, "dense", size_run=False), + pd.DataFrame( + { + "household_id": ids, + "household_weight": np.full(len(ids), 5.0), + "clone_index": np.array([0, 0, 0, 1], dtype=np.int64), + } + ), + ), + label="dense", + ) + # A dense clone of a table-minted spine at 10**19 cannot exist: the + # expand refuses it, and so does the decode, without overflowing. + with pytest.raises(ValueError, match=r"^1 dense row\(s\) carry a clone_index"): + weight_tables(run, id_multiplier=10**19) + floated = replace( + run, weights=run.weights.assign(household_id=lambda t: t.household_id * 1.0) + ) + with pytest.raises(TypeError, match="household_id must be an integer column"): + weight_tables(floated) + + +def test_a_file_that_keeps_its_sources_still_decodes_its_spine_stretch( + tmp_path: Path, +) -> None: + ids = _table_minted_spine_ids() + weights = np.array([10.0, 20.0, 30.0, 40.0]) + run = load_run( + _with_households( + _write_run_dir(tmp_path, "dense", size_run=False), + pd.DataFrame( + { + "household_id": ids, + "household_weight": weights, + "clone_index": np.zeros(len(ids), dtype=np.int64), + "source_household_id": np.array([1, 2, 1, 2], dtype=np.int64), + } + ), + ), + label="dense", + ) + spine = pd.DataFrame({"household_id": ids, "household_weight": weights}) + result = weight_tables(run, spine_weights=spine, n_clones=1) + assert result["source_id_multiplier_basis"] == "source_household_id" + assert result["distinct_source_households"] == 2 + assert result["stretch"]["vs_pool_design"]["max"] == pytest.approx(1.0) + + def test_area_support_and_gate_tables(tmp_path: Path) -> None: run = load_run( _write_run_dir(tmp_path, "candidate", size_run=True), label="candidate" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py index 7e200cb557..f44dd1200a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py @@ -213,8 +213,11 @@ def test_spine_leaves_fail_closed_on_unknown_raw_person(tmp_path: Path) -> None: extra_adult_rows=({"sernum": 9, "person": 1, "inearns": 1.0},), ) - with pytest.raises(ValueError, match="absent from the raw spine"): + with pytest.raises(ValueError, match="absent from the raw spine") as caught: UKFRSHMRCSpineLeavesStageTransform(tmp_path, stage=stage)(_base_frame()) + # The refusal counts the unknown people and names none (microcosm#931). + assert str(caught.value).startswith("Raw FRS retained leaves contain 1 person") + assert "9001" not in str(caught.value) def test_spine_leaves_sampled_rung_restricts_to_surviving_people( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py index 9d609775b0..085e61203e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py @@ -769,13 +769,66 @@ def test_spi_fill_rejects_missing_source_ids() -> None: } ) - with pytest.raises(ValueError, match="missing source ID"): + with pytest.raises(ValueError, match="missing source ID") as caught: fill_support_channel_from_source( result.person, donor, entity="person", columns=["employment_income"], ) + # Person 2002's copy names a source the donor lacks; the refusal counts + # the copy rows and names no id (microcosm#931). + assert str(caught.value).endswith("1 channel row(s) name a source it lacks.") + assert "2002" not in str(caught.value) + + repeated = pd.concat([donor, donor.iloc[[0]]], ignore_index=True) + with pytest.raises(ValueError, match="must be unique") as caught: + fill_support_channel_from_source( + result.person, + repeated, + entity="person", + columns=["employment_income"], + ) + assert "1 row(s) repeat" in str(caught.value) + assert "1001" not in str(caught.value) + + +def test_spi_support_input_refusals_count_rows_and_name_no_id() -> None: + """The support stage's uniqueness, link and selection refusals print counts + (microcosm#931); every id here is synthetic.""" + + person = person_frame() + person.loc[3, "person_household_id"] = 777_777 + with pytest.raises(ValueError, match="absent from household") as caught: + create_uk_spi_support_tables( + person=person, + benunit=benunit_frame(), + household=household_frame(), + spi_household_count=None, + ) + assert "on 1 person row(s)" in str(caught.value) + assert "777777" not in str(caught.value) + + household = household_frame() + household.loc[2, "household_id"] = 2 + with pytest.raises(ValueError, match="must be unique") as caught: + create_uk_spi_support_tables( + person=person_frame(), + benunit=benunit_frame(), + household=household, + spi_household_count=None, + ) + assert "1 row(s) repeat" in str(caught.value) + + with pytest.raises(ValueError, match="absent from household") as caught: + create_uk_spi_support_tables( + person=person_frame(), + benunit=benunit_frame(), + household=household_frame(), + selected_household_ids=(1, 888_888), + ) + assert "contains 1 value(s)" in str(caught.value) + assert "888888" not in str(caught.value) def _pension_age_frames(): diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 9abc56aa01..4fa04edfb2 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -483,7 +483,7 @@ "cgt_residential_probability", "capital_gains_residential_property" ], - "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor reads for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time.", + "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time.", "operations": [ { "artifact_role": "cgt_asset_type_facts", diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index d1a8f98e1c..b3b94c1edf 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -18,6 +18,7 @@ import numpy as np import pandas as pd +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.frs_brma import ( UK_BRMA_DECLARED_SEEDS, _benunit_regions, @@ -38,6 +39,7 @@ uc_age_eligible_benunits, uk_take_up_population_policy, ) +from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE from microcosm.build.uk_runtime.national_frame import ( load_uk_national_frame, uk_build_identifier_frame, @@ -868,7 +870,6 @@ def _e8_support_split( ) person = frame.table("person") - benunit = frame.table("benunit") household = frame.table("household") missing = [ column @@ -905,7 +906,7 @@ def _e8_support_split( float(record.new_total), ] try: - lineage = _support_copy_lineage(person, benunit, household) + lineage = _support_copy_lineage(household) except ValueError as error: problems["support_split_lineage"] = str(error) return receipt @@ -916,7 +917,9 @@ def _e8_support_split( .astype("int64") .to_numpy() ) - receipt["id_multiplier"] = int(lineage.multiplier) + # The declared radix of the split's copy-id field (microcosm#931), no + # longer one read off the frame's largest id. + receipt["id_multiplier"] = int(lineage.radix) receipt["pre_split_households"] = int(pre_split.sum()) receipt["copies"] = int(len(lineage.copy_positions)) # The explicit copy index (microcosm#1045 review): present on artifacts @@ -1125,11 +1128,12 @@ def _e8_residential_split( record.declared_factor, ] try: - lineage = _residential_arm_lineage(person, frame.table("benunit"), household) + lineage = _residential_arm_lineage(household) except ValueError as error: problems["residential_split_lineage"] = str(error) return receipt - receipt["id_multiplier"] = int(lineage.multiplier) + # The declared radix of the arms' copy-id field (microcosm#931). + receipt["id_multiplier"] = int(lineage.radix) receipt["arms"] = int(len(lineage.arm_positions)) receipt["households_split"] = int(np.unique(lineage.source_positions).size) receipt["largest_arm_index"] = ( @@ -1944,6 +1948,10 @@ def main() -> int: # Every draw these receipts replay is keyed on the build's identifiers; a # release carries them beside its renumbered ones (microcosm#1194). frame = uk_build_identifier_frame(frame) + # The receipts decode copy ids at the copy-id table's declared radices + # (microcosm#931); a spine minted by the rule the table replaced is + # refused once, here, before any receipt reads it. + _require_copy_ids_on_the_declared_table(frame.table("household")) if args.check == "e4": from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine @@ -2142,13 +2150,114 @@ def _pre_residential_household_weights(frame) -> np.ndarray: weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() if HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE not in household.columns: return weights - lineage = _residential_arm_lineage( - frame.table("person"), frame.table("benunit"), household - ) + lineage = _residential_arm_lineage(household) np.add.at(weights, lineage.source_positions, weights[lineage.arm_positions]) return weights +# --- copy ids at the copy-id table's declared radices (microcosm#931) -------- +# +# Every UK copy layer mints its copies' ids in a decimal field of its own, +# declared in ``uk_runtime.copy_id_table``: copy ``k`` of a row in field ``f`` +# carries the row's ids plus ``k`` times ``f``'s radix. The receipts decode a +# copy id only through ``_copy_ordinals_and_parents``. A spine built before +# the table offset each layer's copies by a power of ten above the largest id +# the layer saw; ``_require_copy_ids_on_the_declared_table`` refuses such a +# spine, which is receipted with the tool at the commit that built it. +# Decoding that retired rule as well would mean changing these two functions, +# and nothing else. + +#: The old-spine refusal, after its count of household rows. +_OFF_TABLE_REFUSAL = ( + "household row(s) carry ids that do not follow the declared copy-id table " + "(microcosm#931): each copy field of an id must hold the ordinal its row's " + "lineage columns give. A spine built before the table follows the rule the " + "table replaced; receipt it with this tool at the commit that built it." +) + + +def _exact_household_ids(household: pd.DataFrame, *, what: str) -> np.ndarray: + """``household_id`` as exact int64; a non-integer column is refused, not cast.""" + + values = household["household_id"] + if not pd.api.types.is_integer_dtype(values.dtype): + raise TypeError(f"{what} requires integer household ids, not {values.dtype}.") + return values.to_numpy(dtype=np.int64) + + +def _copy_id_rows_off_the_declared_table(household: pd.DataFrame) -> int: + """How many household rows' ids disagree with their copy lineage. + + Each field of the copy-id table holds the ordinal of one copying step of + the UK's copy lineage (``household_lineage.UK_COPY_LINEAGE``): the step's + ordinal column where the artifact carries it, else whether the step's + marker flags the row, else zero, the step not having run. The ``vintage`` + field, which nothing mints yet, holds zero. A row follows the table when + its id holds exactly those ordinals. + """ + + ids = _exact_household_ids(household, what="The copy-id table check") + branches = {branch.name: branch for branch in UK_COPY_LINEAGE.branches} + off = np.zeros(len(ids), dtype=bool) + for field in copy_id_table.UK_COPY_ID_FIELDS: + held = copy_id_table.ordinal(ids, field.name) + branch = branches.get(field.name) + index = None if branch is None else branch.index + marker = None if branch is None else branch.marker + if index is not None and index in household.columns: + ordinals = household[index] + if not pd.api.types.is_integer_dtype(ordinals.dtype): + raise TypeError( + f"The copy-id table check requires integer {index!r} ordinals." + ) + off |= held != ordinals.to_numpy(dtype=np.int64) + elif marker is not None and marker in household.columns: + flagged = household[marker].eq(branch.copy).to_numpy(dtype=bool) + off |= (held != 0) != flagged + else: + off |= held != 0 + return int(off.sum()) + + +def _require_copy_ids_on_the_declared_table(household: pd.DataFrame) -> None: + """Refuse a frame whose copy ids are off the declared table, by count.""" + + off = _copy_id_rows_off_the_declared_table(household) + if off: + raise ValueError(f"{off} {_OFF_TABLE_REFUSAL}") + + +def _copy_ordinals_and_parents( + ids: np.ndarray, field: str +) -> tuple[np.ndarray, np.ndarray]: + """Each copy id's ordinal in ``field`` and the id it was copied from. + + Read at the field's declared radix in the copy-id table, on exact int64. + """ + + return copy_id_table.ordinal(ids, field), copy_id_table.parent(ids, field) + + +def _parent_positions( + household_ids: np.ndarray, + ordinals: np.ndarray, + parents: np.ndarray, + *, + eligible: np.ndarray, +) -> tuple[np.ndarray, int]: + """Each copy's parent's position, and how many copies have none. + + A copy without one has ordinal zero, or a parent id absent from the frame + or outside ``eligible`` (the rows the stage copied). + """ + + positions = pd.Index(household_ids).get_indexer(parents) + found = positions >= 0 + orphaned = (ordinals < 1) | ~found + orphaned[found] |= ~eligible[positions[found]] + return positions, int(orphaned.sum()) + + class _ResidentialArmLineage(NamedTuple): """Where every residential arm sits and which source it came from.""" @@ -2157,34 +2266,29 @@ class _ResidentialArmLineage(NamedTuple): arm_positions: np.ndarray source_positions: np.ndarray arm_index: np.ndarray - multiplier: int + radix: int -def _residential_arm_lineage( - person: pd.DataFrame, benunit: pd.DataFrame, household: pd.DataFrame -) -> _ResidentialArmLineage: +def _residential_arm_lineage(household: pd.DataFrame) -> _ResidentialArmLineage: """Every residential arm's position, its source's position and its index. - The split offsets ids by ``id_multiplier_for_values`` over the rows it - saw, every household of the frame before it ran: the households whose - arm flag is false. Arm ``j`` of source ``r`` carries ``r + j * M``, so - ``j = id // M`` and ``r = id - j * M``. An arm without a source, or with - index zero, fails closed, and the stored ``cgt_residential_clone_index`` - must agree with the id scheme on every row. + Arm ``j`` of source ``r`` carries ``r + j x R`` on every id, ``R`` the + declared radix of the copy-id table's ``cgt_residential_split`` field + (microcosm#931), so ``j`` is the id's ordinal in that field and ``r`` the + id with the field cleared; the split's sources are the households whose + arm flag is false. An arm without a source, or with index zero, fails + closed, and the stored ``cgt_residential_clone_index`` must agree with the + id on every arm. The refusals print counts, never an id. """ from microcosm.build.uk_runtime.cgt_residential_split import ( CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, + CGT_RESIDENTIAL_COPY_ID_FIELD, HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, ) - from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values - household_ids = ( - pd.to_numeric(household["household_id"], errors="raise") - .astype("int64") - .to_numpy() - ) - if len(np.unique(household_ids)) != len(household_ids): + household_ids = _exact_household_ids(household, what="CGT residential-arm lineage") + if not pd.Index(household_ids).is_unique: raise ValueError("CGT residential-arm lineage requires unique household ids.") is_arm = household[HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE].to_numpy(dtype=bool) if CGT_RESIDENTIAL_CLONE_INDEX_COLUMN not in household.columns: @@ -2203,41 +2307,25 @@ def _residential_arm_lineage( f"{flag_disagreements} household(s)." ) pre_split = ~is_arm - pre_split_ids = set(household_ids[pre_split].tolist()) - person_pre_split = person["person_household_id"].isin(pre_split_ids).to_numpy() - benunit_pre_split = ( - benunit["benunit_id"] - .isin(set(person.loc[person_pre_split, "person_benunit_id"].tolist())) - .to_numpy() + arm_positions = np.flatnonzero(is_arm) + arm_index, source_ids = _copy_ordinals_and_parents( + household_ids[arm_positions], CGT_RESIDENTIAL_COPY_ID_FIELD ) - multiplier = id_multiplier_for_values( - person.loc[person_pre_split, "person_id"], - person.loc[person_pre_split, "person_household_id"], - person.loc[person_pre_split, "person_benunit_id"], - benunit.loc[benunit_pre_split, "benunit_id"], - household_ids[pre_split], + source_positions, orphaned = _parent_positions( + household_ids, arm_index, source_ids, eligible=pre_split ) - arm_positions = np.flatnonzero(is_arm) - arm_ids = household_ids[arm_positions] - arm_index = arm_ids // multiplier - source_ids = arm_ids - arm_index * multiplier - position_by_id = {int(value): index for index, value in enumerate(household_ids)} - source_positions = np.empty(len(arm_positions), dtype=np.int64) - for slot, (arm_id, index, source_id) in enumerate( - zip(arm_ids.tolist(), arm_index.tolist(), source_ids.tolist(), strict=True) - ): - source = position_by_id.get(int(source_id)) - if index < 1 or source is None or not pre_split[source]: - raise ValueError( - "CGT residential arm without a source household: household_id " - f"{arm_id} (id multiplier {multiplier})." - ) - source_positions[slot] = source + if orphaned: + raise ValueError( + f"{orphaned} CGT residential arm(s) without a source household at the " + "declared radix of the copy-id table's " + f"{CGT_RESIDENTIAL_COPY_ID_FIELD!r} field." + ) scheme_disagreements = int((stored_index[arm_positions] != arm_index).sum()) if scheme_disagreements: raise ValueError( f"CGT residential arm index stored on {scheme_disagreements} arm(s) " - f"disagrees with the id scheme (id // {multiplier})." + "disagrees with the id scheme (the id's ordinal in the copy-id " + f"table's {CGT_RESIDENTIAL_COPY_ID_FIELD!r} field)." ) return _ResidentialArmLineage( household_ids=household_ids, @@ -2245,7 +2333,7 @@ def _residential_arm_lineage( arm_positions=arm_positions, source_positions=source_positions, arm_index=arm_index, - multiplier=multiplier, + radix=copy_id_table.radix(CGT_RESIDENTIAL_COPY_ID_FIELD), ) @@ -2257,45 +2345,41 @@ class _SupportCopyLineage(NamedTuple): copy_positions: np.ndarray root_positions: np.ndarray copy_index: np.ndarray - multiplier: int + radix: int -def _support_copy_lineage( - person: pd.DataFrame, benunit: pd.DataFrame, household: pd.DataFrame -) -> _SupportCopyLineage: +def _support_copy_lineage(household: pd.DataFrame) -> _SupportCopyLineage: """Every support copy's position, its root's position and its index ``k``. - The split offsets ids by ``id_multiplier_for_values`` over the rows it - saw, and those rows are recoverable from the artifact: the households - whose copy flag is false, the clone stage's rows excluded (the split runs - before the clone, so a clone of a copy carries the copy flag but is not a - pre-clone row). Copy ``k`` of root ``r`` carries ``r + k * M1``, so ``k = - id // M1`` and ``r = id - k * M1``. A copy without a root, or with index - zero, fails closed. An artifact that carries the split's explicit + Copy ``k`` of root ``r`` carries ``r + k x R`` on every id, ``R`` the + declared radix of the copy-id table's ``cgt_support_split`` field + (microcosm#931), so ``k`` is the id's ordinal in that field and ``r`` the + id with the field cleared. The split's roots are recoverable from the + artifact: the households whose copy flag is false, the clone stage's rows + excluded (the split runs before the clone, so a clone of a copy carries + the copy flag but is not a pre-clone row). A copy without a root, or with + index zero, fails closed. An artifact that carries the split's explicit ``cgt_support_copy_index`` (0 on roots and unsplit households, ``k`` on copy ``k``; the lineage the geography identity keys on) must agree with - the id scheme on every row, or the lineage fails closed too. + the id on every copy, or the lineage fails closed too. The refusals print + counts, never an id. """ from microcosm.build.uk_runtime.cgt_structure import HOUSEHOLD_IS_CGT_CLONE from microcosm.build.uk_runtime.cgt_support import ( + CGT_SUPPORT_COPY_ID_FIELD, CGT_SUPPORT_COPY_INDEX_COLUMN, HOUSEHOLD_IS_CGT_SUPPORT_COPY, ) - from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values - household_ids = ( - pd.to_numeric(household["household_id"], errors="raise") - .astype("int64") - .to_numpy() - ) - if len(np.unique(household_ids)) != len(household_ids): + household_ids = _exact_household_ids(household, what="CGT support-copy lineage") + if not pd.Index(household_ids).is_unique: raise ValueError("CGT support-copy lineage requires unique household ids.") is_copy = household[HOUSEHOLD_IS_CGT_SUPPORT_COPY].to_numpy(dtype=bool) stored_index = None if CGT_SUPPORT_COPY_INDEX_COLUMN in household.columns: # The explicit lineage is authoritative: the flag and the stored index - # must agree before any id arithmetic is read. + # must agree before any id is decoded. stored_index = ( pd.to_numeric(household[CGT_SUPPORT_COPY_INDEX_COLUMN], errors="raise") .astype("int64") @@ -2313,42 +2397,26 @@ def _support_copy_lineage( else np.ones(len(household), dtype=bool) ) pre_split = pre_clone & ~is_copy - pre_split_ids = set(household_ids[pre_split].tolist()) - person_pre_split = person["person_household_id"].isin(pre_split_ids).to_numpy() - benunit_pre_split = ( - benunit["benunit_id"] - .isin(set(person.loc[person_pre_split, "person_benunit_id"].tolist())) - .to_numpy() + copy_positions = np.flatnonzero(pre_clone & is_copy) + copy_index, root_ids = _copy_ordinals_and_parents( + household_ids[copy_positions], CGT_SUPPORT_COPY_ID_FIELD ) - multiplier = id_multiplier_for_values( - person.loc[person_pre_split, "person_id"], - person.loc[person_pre_split, "person_household_id"], - person.loc[person_pre_split, "person_benunit_id"], - benunit.loc[benunit_pre_split, "benunit_id"], - household_ids[pre_split], + root_positions, orphaned = _parent_positions( + household_ids, copy_index, root_ids, eligible=pre_split ) - copy_positions = np.flatnonzero(pre_clone & is_copy) - copy_ids = household_ids[copy_positions] - copy_index = copy_ids // multiplier - root_ids = copy_ids - copy_index * multiplier - position_by_id = {int(value): index for index, value in enumerate(household_ids)} - root_positions = np.empty(len(copy_positions), dtype=np.int64) - for slot, (copy_id, index, root_id) in enumerate( - zip(copy_ids.tolist(), copy_index.tolist(), root_ids.tolist(), strict=True) - ): - root = position_by_id.get(int(root_id)) - if index < 1 or root is None or not pre_split[root]: - raise ValueError( - "CGT support copy without a root household: household_id " - f"{copy_id} (id multiplier {multiplier})." - ) - root_positions[slot] = root + if orphaned: + raise ValueError( + f"{orphaned} CGT support copy(ies) without a root household at the " + "declared radix of the copy-id table's " + f"{CGT_SUPPORT_COPY_ID_FIELD!r} field." + ) if stored_index is not None: scheme_disagreements = int((stored_index[copy_positions] != copy_index).sum()) if scheme_disagreements: raise ValueError( f"CGT support copy index stored on {scheme_disagreements} " - f"copy(ies) disagrees with the id scheme (id // {multiplier})." + "copy(ies) disagrees with the id scheme (the id's ordinal in the " + f"copy-id table's {CGT_SUPPORT_COPY_ID_FIELD!r} field)." ) return _SupportCopyLineage( household_ids=household_ids, @@ -2356,7 +2424,7 @@ def _support_copy_lineage( copy_positions=copy_positions, root_positions=root_positions, copy_index=copy_index, - multiplier=multiplier, + radix=copy_id_table.radix(CGT_SUPPORT_COPY_ID_FIELD), ) @@ -2381,9 +2449,7 @@ def _pre_split_household_weights(frame) -> np.ndarray: household = frame.table("household") if HOUSEHOLD_IS_CGT_SUPPORT_COPY not in household.columns: return weights - lineage = _support_copy_lineage( - frame.table("person"), frame.table("benunit"), household - ) + lineage = _support_copy_lineage(household) np.add.at(weights, lineage.root_positions, weights[lineage.copy_positions]) return weights From b07e5b3f0377fef2158e7d62132f2eff55617ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:35:16 +0000 Subject: [PATCH 016/108] Follow up the review of the copy-id decoders (#931) Three reviews of 50b7ea13d found no must. This commit takes their should findings and the cheap nits, and corrects four claims 50b7ea13d made. - Bindings: stage 14 (frs_hmrc_spine_leaves) draws its employer pension contributions on keyed uniforms through employer_pension_contributions, but did not bind stochastic_assignment, because the derived test read direct imports only. It now binds it, and the test follows one more step: a stage draws when its module or a bound helper imports the helpers, or imports another first-party module's function that draws through them (a fixed point over the import graph, re-exports and module attributes included). spi_support_channel, which shares spi_spine with stage 14's draw and stage 18's SPI imputation but calls neither, is the one exemption, pinned to those two names. employer_pension_contributions itself stays unbound until the wider binding sweep. - Size evaluation: the spine-stretch refusal printed no count and chained a pandas KeyError listing every missing household id; it now counts the dense rows before the lookup and raises unchained. The misfit refusal says an inferred multiplier reads household ids only and names weight_tables(id_multiplier=), the one route for a recorded multiplier. - Count-only refusals: the row-wise tool's dry-run input checks now call the build's own (rowwise_dataset._require_unique and _require_person_links) instead of listing up to five ids, and the spine sampler's three refusals in national_sampling count families. - Guards: an AST guard refuses ten raised to a len(str(...)) power outside rowwise_geography (the dense rule), copy_id_table (field spans) and national_sampling's legacy candidate decoder, so the retired rule cannot come back written out inline. - Texts, correcting 50b7ea13d: - a retired-rule decode would not replace two functions and nothing else; it would also thread the person and benefit-unit tables back through the two lineage functions and their four callers, and pair the twins in the tool at the three pair_clone_households call sites (the tool's comment now says so); - only the identity tool used the retired rule; size evaluation kept the dense line's own rule and overflowed at 10**19, and its non-integer id refusal names the dtype, not a count; - no caller passes a recorded dense multiplier yet: tools/evaluate_uk_dataset_size.py infers it from household ids (the docstrings say so); - the arm tests decoded a synthetic copy chain, not stage output; the stage-built tests below cover that now. The changelog names what each SPI and CGT stage binds and why, limits the count-only sentence to the UK modules (the shared graph executor's EXPAND checks are unchanged), describes the inferred multiplier, and says the identity tool refuses a pre-table spine with copy rows. What moves: no data value or id. Stage 14's key moves with its new binding; every spine node re-keys anyway through graph_kernels.py, and the dense population nodes that hash national_sampling.py re-key, as they already do through rowwise_dataset.py in this push. No generated file changes. Tests: new size tests decode an older dense run at K=15 on odd ids above 2**53 at all three bases (a float64 subtraction fails it), and pin the uncovered-spine refusal (no id in the message, its cause or the traceback) and the negative clone_index refusal. New identity-tool tests hold the old-spine gate to each stored ordinal exactly and to an empty vintage field, refuse a support copy whose parent the split did not copy, and run the gate on frames that stages 16, 17, 26, 42, 43, 45 and 46 built (stage 44's redraw done by hand) and the E8 residential, clone-pair and support-split receipts on the last of them. The dry run, the sampler, the CGT carrier and redraw checks, the SPI uniqueness checks and the support-copy refusals now have exact count-only messages pinned. Synthetic mutations make each new guard and test fail. The 30 touched engine-free test files pass (1,331 tests). The coverage manifest's check passes, the H2 fixture regenerates byte-identically, and H2 parity and the engine graph tests pass in the engine slot. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-copy-id-table.changed.md | 6 +- .../build/uk_runtime/graph_kernels.py | 16 +- .../build/uk_runtime/national_sampling.py | 14 +- .../build/uk_runtime/size_evaluation.py | 46 ++-- .../engine_free/uk/test_uk_cgt_imputation.py | 22 ++ .../engine_free/uk/test_uk_cgt_support.py | 4 +- .../engine_free/uk/test_uk_copy_id_table.py | 56 +++++ .../tests/engine_free/uk/test_uk_graph.py | 217 ++++++++++++++++-- .../uk/test_uk_high_value_home_support.py | 20 ++ .../uk/test_uk_identity_stability_receipts.py | 205 ++++++++++++++++- .../uk/test_uk_national_sampling.py | 44 +++- .../engine_free/uk/test_uk_rowwise_dry_run.py | 56 ++++- .../engine_free/uk/test_uk_size_evaluation.py | 92 ++++++++ .../engine_free/uk/test_uk_spi_band_donors.py | 4 + .../engine_free/uk/test_uk_spi_support.py | 9 +- tools/build_uk_rowwise_dataset.py | 40 +--- tools/verify_uk_identity_stability.py | 20 +- 17 files changed, 773 insertions(+), 98 deletions(-) diff --git a/changelog.d/931-uk-copy-id-table.changed.md b/changelog.d/931-uk-copy-id-table.changed.md index 10fb12ef16..2fa3d56fff 100644 --- a/changelog.d/931-uk-copy-id-table.changed.md +++ b/changelog.d/931-uk-copy-id-table.changed.md @@ -1,6 +1,6 @@ The UK spine mints every copy row's ids from fixed decimal fields declared in `microcosm.build.uk_runtime.copy_id_table`, not from a power of ten above the largest id each copy layer sees (#931). Once the spine clones households at its start, that largest id would grow with the clone count K, so every layer's offset would move with K and the K=1 spine would stop being the clone-0 slice of the K=15 spine. Copy `k` in a field takes `id + k × radix` on all five id columns, so every copy id is a function of its source id and its copy path alone. The fields, least significant first: `vintage` (10 ordinals at 10^8, reserved for pooled FRS vintages, minted by nothing yet), `geographic_support` (100 at 10^9, for the clone-first spine), `spi_support_channel` (3 at 10^11: the SPI copy at 1, the income band donor at 2), `high_value_support` (6 at 10^12), `cgt_support_split` (10,000 at 10^13), `cgt_incidence_clone` (2 at 10^17) and `cgt_residential_split` (8 at 10^18, the top field). The largest mintable id is about 7.2e18, below int64's limit, which the module checks when it loads. -- **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 on any frame; the row-by-row walk it replaces read a row holding only integer and float columns as float64, inexact above 2^53. Refusals on the copy path print counts, never ids: the new ones, and the older checks that listed up to five ids or printed the largest (uniqueness, links, sources, carriers and weights in the clone helpers, the SPI support and spine stages, the row-wise dataset and the CGT stages). `create_uk_spi_support_tables` no longer takes an `id_multiplier`. The identity tool (`tools/verify_uk_identity_stability.py`) decodes the CGT support copies and the residential arms at their fields' declared radices and records the declared radix as each receipt's `id_multiplier`; it refuses, by count, a spine whose copy ids do not follow the table, which every spine built before the table is, and such a spine is receipted with the tool at the commit that built it. +- **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 on any frame; the row-by-row walk it replaces read a row holding only integer and float columns as float64, inexact above 2^53. The UK modules' refusals on the copy path print counts, never ids: the new ones, and the older checks that listed up to five ids or printed the largest (uniqueness, links, sources, carriers and weights in the clone helpers, the SPI support and spine stages, the row-wise dataset and its tool's dry-run plans, the CGT stages and the spine sampler); the shared graph executor's own EXPAND checks, which every copy stage also passes through, are unchanged, and some of them still name ids. `create_uk_spi_support_tables` no longer takes an `id_multiplier`. The identity tool (`tools/verify_uk_identity_stability.py`) decodes the CGT support copies and the residential arms at their fields' declared radices and records the declared radix as each receipt's `id_multiplier`; it refuses, by count, a spine whose copy ids do not follow the table, as every pre-table spine with copy rows does, and such a spine is receipted with the tool at the commit that built it. - **What moves.** Base FRS rows keep their ids. Every copy row's five id columns now sit at its parent row's ids plus its layer's offset: SPI copies 10^11, band donors 2×10^11, high-value copy k k×10^12, CGT copy k k×10^13, twins 10^17 and arm j j×10^18. Each used to be a power of ten above the largest id its layer saw. Relative id order is unchanged at K=1, so sorts, ties, ranks and folds are unchanged. Values move on copy rows through every draw keyed on the copy's own id: the SPI income stage's second stage, stages 19 to 24 and 27 to 41, every twin's Advani-Summers prior (so the CGT gains layer is re-realised), stages 48 to 54 and policyengine-uk's Warm Home Discount draw on copy households in spine engine calls. Base rows move second-order through pooled steps: the high-value selection, the regional value ranks, the landlord walk, the residential solve, the anchor, the CGT walks and calibration. Draws keyed on `person_source_id` do not move. National ids pass 2^53 from the incidence twin on, and at the CGT support split for any copy at ordinal 901 or more. The released file's identifiers, ranks of the build ids, do not move at K=1, while its `*_build_id` columns carry the new copy ids, so joining copy rows across versions goes through lineage (source id and copy path), not build ids. -- **Dense line.** The dense expand keeps its multiplier above the spine's largest id. A spine carrying residential arms (multiples of 10^18), as a production national spine does, leaves no int64 room for a second dense clone, so every dense entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) refuses wherever the dense offset would overflow int64, in one message naming #931 and the clone-first spine. Dense builds with K > 1 of such a spine wait for #931's clone-first spine; K = 1, and a spine whose ids leave room (one without copy rows, say), expand as before. The dataset-size evaluation decodes dense ids back to spine ids on exact integers, at the multiplier the expansion recorded (`weight_tables(id_multiplier=)`), the spine's or the one the clone-0 ids give, 10^19 included, and refuses by count a dataset its multiplier does not fit; a dense file that keeps its true sources decodes its spine stretch again instead of failing. -- **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes, and the SPI support, band-donor, high-value and CGT clone and anchor stages also bind the helpers that mint or weigh their copies (`spi_support`, `cgt_support`, `rowwise_geography`). Every stage whose module or bound helpers draw through `microcosm.build.stochastic_assignment` now binds it, so an edit to the keyed draws re-keys each stage they move. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. +- **Dense line.** The dense expand keeps its multiplier above the spine's largest id. A spine carrying residential arms (multiples of 10^18), as a production national spine does, leaves no int64 room for a second dense clone, so every dense entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) refuses wherever the dense offset would overflow int64, in one message naming #931 and the clone-first spine. Dense builds with K > 1 of such a spine wait for #931's clone-first spine; K = 1, and a spine whose ids leave room (one without copy rows, say), expand as before. The dataset-size evaluation decodes dense ids back to spine ids on exact integers, 10^19 included. A caller can pass the multiplier the expansion recorded (`weight_tables(id_multiplier=)`); `tools/evaluate_uk_dataset_size.py` infers it from the spine's household ids or the clone-0 household ids, and a dataset the inferred multiplier does not fit is refused by count. A dense file that keeps its true sources decodes its spine stretch again instead of failing. +- **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes. The SPI support and band-donor stages also bind `spi_support`, whose clone helper mints their copies; the high-value split binds `cgt_support`, whose exact-total correction weighs its copies; and the CGT clone and anchor stages, which share `cgt_structure`, bind `rowwise_geography`, whose clone helper mints the twins. The exact-total weighting in `spi_support`, which the twin and the high-value split reach, stays unbound until a wider binding sweep. Every stage whose module or bound helpers import `microcosm.build.stochastic_assignment`, or import another module's function that draws through it, now binds it, so an edit to the keyed draws re-keys each stage they move; the SPI support channel, which shares its module with stage 14's employer pension draw but never calls it, is exempt, and the modules in between (`employer_pension_contributions`, say) are not bound. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 9cf94c2346..21f0174005 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -172,10 +172,14 @@ # Imported modules are not traversed by ``source_hash``. Bind relationship # helpers and the adapter's input-retention checks into every consuming stage. -# Every stage whose module or bound helpers draw through the shared keyed-draw -# helpers (``microcosm.build.stochastic_assignment``) binds them, so an edit to -# how a uniform is keyed or a binary is placed re-keys every stage it moves -# (microcosm#931); test_uk_graph derives the set from the imports. +# Every stage whose module or bound helpers import the shared keyed-draw +# helpers (``microcosm.build.stochastic_assignment``), or import another +# module's function that draws through them, binds them, so an edit to how a +# uniform is keyed or a binary is placed re-keys every stage it moves +# (microcosm#931). test_uk_graph derives the set from the imports and names, +# with its reason, each stage that imports a drawing function it never calls. +# The intermediate modules themselves (employer_pension_contributions, say) +# are not bound here. _STAGE_HELPER_MODULES = { # The three FRS draw stages' receipts measure their take-up flags through # frs_take_up's signal measurement and count support by the shared @@ -183,7 +187,9 @@ "frs_take_up": (receipt_counts, stochastic_assignment), "frs_person_draws": (frs_take_up, receipt_counts, stochastic_assignment), "frs_household_draws": (frs_take_up, receipt_counts, stochastic_assignment), - "frs_hmrc_spine_leaves": (frs_hmrc_source,), + # The leaves' employer pension contributions are drawn on keyed uniforms + # through employer_pension_contributions (microcosm#931). + "frs_hmrc_spine_leaves": (frs_hmrc_source, stochastic_assignment), # The annual income descriptors' receipt follows the shared disclosure rules # (microcosm#1164). "frs_annual_income_spells": (receipt_counts, stochastic_assignment), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py index 1b130f25d9..67a30316ca 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py @@ -215,12 +215,11 @@ def uk_spine_source_family_units(frame: Frame) -> tuple[np.ndarray, np.ndarray]: ) source_rows = raw & (household_ids == units) source_keys = set(int(value) for value in units[source_rows]) - missing_sources = sorted(set(int(value) for value in units) - source_keys) + missing_sources = len(set(int(value) for value in units) - source_keys) if missing_sources: raise ValueError( - "UK spine sample: source_household_id family key(s) lack a " - "household_id-bearing raw row: " - f"{missing_sources[:5]}." + f"UK spine sample: {missing_sources} source_household_id family " + "key(s) lack a household_id-bearing raw row." ) raw_region_by_unit: dict[int, str] = {} @@ -233,8 +232,8 @@ def uk_spine_source_family_units(frame: Frame) -> tuple[np.ndarray, np.ndarray]: raw_region_by_unit[unit] = next(iter(family_regions)) if multi_region: raise ValueError( - "UK spine sample: source family spans more than one raw-row " - f"region for key(s) {multi_region[:5]}." + f"UK spine sample: {len(multi_region)} source family(ies) span more " + "than one raw-row region." ) strata = np.asarray( @@ -285,8 +284,7 @@ def sample_uk_spine_frame( ) if incomplete: raise ValueError( - "UK spine sample retained incomplete source family key(s): " - f"{incomplete[:5]}." + f"UK spine sample retained {len(incomplete)} incomplete source family(ies)." ) validate_uk_national_frame(normalized) return normalized, { diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py index 2074068da6..09b1961230 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_evaluation.py @@ -514,13 +514,20 @@ def _source_multiplier( """The dense clone multiplier of ``weights``' ids, and where it was read. The dense expand offsets clone ``c`` of a spine household by ``c`` times - ten to the digit count of the spine's largest id (the dense line's own - rule, ``id_multiplier_for_values``, until microcosm#931 (c) re-bases it - onto the clone-first spine). A multiplier the dataset recorded (the - expansion artifact's or the row-wise manifest's ``id_multiplier``) is - taken as it is; otherwise it is read off the spine, or off the clone-0 - ids. A Python integer: a K=1 expansion of a spine minted from the copy-id - table records 10**19, which int64 cannot hold. + ten to the digit count of the largest of the spine's five id columns (the + dense line's own rule, ``id_multiplier_for_values``, until microcosm#931 + (c) re-bases it onto the clone-first spine). A multiplier a caller passes + (``weight_tables(id_multiplier=)``: what the expansion artifact or the + row-wise manifest recorded) is taken as it is. Otherwise it is inferred + from household ids alone, the spine's or the clone-0 rows'. The spine's + agrees with the expansion's whenever the spine carries copy rows, and the + clone-0 rows' does too when one of them has as many digits as the spine's + largest household id. On a spine whose largest id is a person or + benefit-unit id with more digits (one without copy rows, say), or a + selection whose clone-0 rows are all shorter, the inferred multiplier is + too small and ``_spine_ids`` refuses it by count. A Python integer: a K=1 + expansion of a spine minted from the copy-id table records 10**19, which + int64 cannot hold. """ if recorded is not None: @@ -575,8 +582,9 @@ def _spine_ids(weights: pd.DataFrame, multiplier: int) -> np.ndarray: if off: raise ValueError( f"{off} dense row(s) do not sit at their clone_index times the dense " - "id multiplier above a spine id below it; the multiplier is not this " - "dataset's (pass the id_multiplier its expansion recorded)" + "id multiplier above a spine id below it, so the multiplier is not " + "this dataset's. An inferred multiplier reads household ids only; " + "weight_tables(id_multiplier=) takes the one the expansion recorded." ) return spine_ids @@ -604,8 +612,11 @@ def weight_tables( """Summarize shipped weights, source support, and design stretch. Dense ids decode back to spine ids at the dense clone multiplier: the one - ``id_multiplier`` names (what the dataset's expansion recorded), else the - spine's, else the one the clone-0 ids give (``_source_multiplier``). + ``id_multiplier`` names (what the dataset's expansion recorded), else one + inferred from the spine's household ids, else from the clone-0 household + ids (``_source_multiplier``). No caller passes ``id_multiplier`` yet: + ``tools/evaluate_uk_dataset_size.py`` and ``summarize`` infer it, and a + dataset the inferred multiplier does not fit is refused by count. """ manifest_weights = _mapping(run.manifest.get("weights")) @@ -686,12 +697,15 @@ def weight_tables( ) spine_ids = _spine_ids(weight_frame, multiplier) source_map = spine_weights.set_index("household_id")["household_weight"] - try: - design = source_map.loc[spine_ids].to_numpy(dtype=np.float64) / clones - except KeyError as exc: + # Counted before the lookup: a failed .loc raises a KeyError that + # lists every missing id, and chaining it would print them. + uncovered = int((~pd.Index(spine_ids).isin(source_map.index)).sum()) + if uncovered: raise ValueError( - "spine weights do not cover every source household" - ) from exc + "spine weights do not cover the spine household of " + f"{uncovered} dense row(s)" + ) + design = source_map.loc[spine_ids].to_numpy(dtype=np.float64) / clones ratios = weight_frame["household_weight"].to_numpy(dtype=np.float64) / design summary = _ratio_summary(ratios) recorded = _optional_float( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py index ced02e0066..06d71dde44 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py @@ -430,6 +430,28 @@ def test_summary_reports_achieved_against_published(self) -> None: evidence = summary.evidence() assert {"rows", "age_rows", "region_rows", "age_by_band_rows"} <= set(evidence) + def test_a_person_in_a_household_with_no_weight_is_refused_by_count( + self, monkeypatch + ) -> None: + frame = _frame(2, gains=[1_000.0, 2_000.0], incomes=[20_000.0, 30_000.0]) + # A frame cannot be built with a person naming no household, so the + # stored table is edited in place and the seam's revalidation skipped. + frame.table("person").loc[1, "person_household_id"] = 434_343 + monkeypatch.setattr( + cgt_imputation, "validate_uk_national_frame", lambda frame: None + ) + with pytest.raises(ValueError) as caught: + impute_uk_capital_gains_with_report( + frame, + _distribution(cell_people=10.0), + PARAMETERS, + conditioning=load_hmrc_cgt_conditioning_facts(), + ) + # The refusal counts the households and names none (microcosm#931). + assert str(caught.value) == ( + "Person rows reference 1 household(s) with no weight." + ) + class TestWealthRanking: """Gainers take amounts in the order of a prior and wealth rank blend (#1014).""" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index d13702fc2b..d2002f4819 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -274,7 +274,9 @@ def test_income_band_refuses_a_household_without_an_adult_or_a_missing_component person, household_ids=household["household_id"], parameters=PARAMETERS ) # The refusal counts the households and names none (microcosm#931). - assert str(caught.value).endswith("1 household(s) have none.") + assert str(caught.value) == ( + "Every CGT household requires an adult carrier; 1 household(s) have none." + ) person, _, household, _ = support_tables(weights=[1.0], wealth=[0.0], incomes=[1.0]) with pytest.raises(ValueError, match="components missing"): cgt_support_income_band( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py index ae65724301..b47733453a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -744,6 +744,62 @@ def test_only_the_dense_line_names_the_retired_id_rule() -> None: ) +#: The modules that compute ten to an id's digit count inline, each with what +#: it computes. A module reimplementing the retired rule without naming it +#: (``10 ** len(str(max(ids)))``) fails the guard below. +_INLINE_DIGIT_COUNT_MODULES = { + "packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_geography.py": ( + "defines the dense clone multiplier, id_multiplier_for_values" + ), + "packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py": ( + "sizes each declared field's span from its ordinal bound" + ), + "packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py": ( + "uk_source_family_units, the legacy candidate decoder, re-derives the " + "retired stage fence's clone multiplier; whether it learns the table " + "is open (microcosm#931)" + ), +} + + +def _computes_ten_to_a_digit_count(source: str) -> bool: + """Whether ``source`` raises ten to a power built from ``len(str(...))``.""" + + for node in ast.walk(ast.parse(source)): + if not ( + isinstance(node, ast.BinOp) + and isinstance(node.op, ast.Pow) + and isinstance(node.left, ast.Constant) + and node.left.value == 10 + ): + continue + for call in ast.walk(node.right): + if ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Name) + and call.func.id == "len" + and call.args + and isinstance(call.args[0], ast.Call) + and isinstance(call.args[0].func, ast.Name) + and call.args[0].func.id == "str" + ): + return True + return False + + +def test_only_the_declared_modules_compute_ten_to_an_ids_digit_count() -> None: + """The name guards cannot see the retired rule written out inline, as size + evaluation's clone-0 path was before #931; this one can.""" + + found = { + path + for path, source in _uk_python_sources().items() + if _computes_ten_to_a_digit_count(source) + } + assert found == set(_INLINE_DIGIT_COUNT_MODULES) + assert not set(_INLINE_DIGIT_COUNT_MODULES) & _COPY_ID_MODULES + + def test_no_module_that_mints_or_decodes_copy_ids_names_the_retired_rule() -> None: """Each module that mints, bounds or decodes spine copy ids reads the copy-id table, the identity tool among them, and none names the rule the diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 1fb058a125..323d90ede9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -1,7 +1,9 @@ """Tests split from packages/microcosm-build/tests/test_uk_graph.py.""" # ruff: noqa: F403, F405 +import functools import hashlib +from typing import NamedTuple from microcosm.graph import graph_schema, orrery_document from microcosm.graph.canonical import canonical_json @@ -420,44 +422,214 @@ def changed_helper_bytes(path: Path) -> bytes: assert registry.implementation_hash(ref) == before[ref], ref -#: Stages whose module or bound helpers import the keyed-draw helpers -#: (``microcosm.build.stochastic_assignment``) but need not bind them, each with -#: its reason. None today: every stage that imports them draws through them. -_STOCHASTIC_ASSIGNMENT_UNBOUND: dict[str, str] = {} +_STOCHASTIC_ASSIGNMENT = "microcosm.build.stochastic_assignment" +#: Stages that import a function drawing through the keyed-draw helpers +#: (``microcosm.build.stochastic_assignment``) but never call it, so need not +#: bind them: the drawing names each imports, and why it never calls them. +_STOCHASTIC_ASSIGNMENT_UNBOUND: dict[str, tuple[frozenset[str], str]] = { + "spi_support_channel": ( + frozenset( + { + "employer_pension_contributions.draw_employer_pension_contributions", + "spi_income.impute_uk_spi_income_support", + } + ), + "its module, spi_spine, also holds frs_hmrc_spine_leaves' employer " + "pension draw and hmrc_spi_income_spine's SPI income imputation, both " + "of which bind the helpers; the support channel's transform calls " + "neither", + ), +} + + +class _DrawFacts(NamedTuple): + """What one first-party module imports and what its definitions name.""" + + imports: tuple[tuple[str, str, str], ...] + module_aliases: dict[str, str] + references: dict[str, frozenset[str]] + + +@functools.cache +def _draw_facts(name: str) -> _DrawFacts | None: + import ast + import importlib.util + + if not name.startswith("microcosm.") or name == _STOCHASTIC_ASSIGNMENT: + return None + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): + return None + if spec is None or not (spec.origin or "").endswith(".py"): + return None + package = name if spec.submodule_search_locations else name.rpartition(".")[0] + tree = ast.parse(Path(spec.origin).read_text(encoding="utf-8")) + imports: list[tuple[str, str, str]] = [] + module_aliases: dict[str, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + base = package.rsplit(".", node.level - 1)[0] if node.level else "" + origin = ( + ".".join(part for part in (base, node.module) if part) + if node.level + else node.module or "" + ) + for alias in node.names: + local = alias.asname or alias.name + imports.append((local, origin, alias.name)) + module_aliases[local] = f"{origin}.{alias.name}" + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.asname: + module_aliases[alias.asname] = alias.name + references = {} + for node in tree.body: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + names = set() + for sub in ast.walk(node): + if isinstance(sub, ast.Name): + names.add(sub.id) + elif isinstance(sub, ast.Attribute) and isinstance(sub.value, ast.Name): + names.add(f"{sub.value.id}.{sub.attr}") + references[node.name] = frozenset(names) + return _DrawFacts(tuple(imports), module_aliases, references) + + +def _drawing_names(roots) -> dict[str, frozenset[str]]: + """Each first-party module's names that draw through the keyed-draw helpers. + + A definition draws when it names one of their functions or another drawing + name, of its own module or imported from (or reached by attribute on) + another; a module's drawing names include the drawing names it imports, so + a re-export draws too. A fixed point over every module the roots reach. + """ + facts: dict[str, _DrawFacts] = {} + pending = list(roots) + while pending: + name = pending.pop() + if name in facts: + continue + found = _draw_facts(name) + if found is None: + continue + facts[name] = found + pending.extend(origin for _, origin, _ in found.imports) + pending.extend(found.module_aliases.values()) + drawing: dict[str, set[str]] = {name: set() for name in facts} + + def draws(module: str, local: str) -> bool: + info = facts[module] + if local in drawing[module]: + return True + if "." in local: + head, _, attribute = local.partition(".") + target = info.module_aliases.get(head) + return target == _STOCHASTIC_ASSIGNMENT or attribute in drawing.get( + target, () + ) + return any( + local == alias + and ( + origin == _STOCHASTIC_ASSIGNMENT or imported in drawing.get(origin, ()) + ) + for alias, origin, imported in info.imports + ) + + changed = True + while changed: + changed = False + for module, info in facts.items(): + for local, _, _ in info.imports: + if local not in drawing[module] and draws(module, local): + drawing[module].add(local) + changed = True + for definition, names in info.references.items(): + if definition not in drawing[module] and any( + draws(module, name) for name in names + ): + drawing[module].add(definition) + changed = True + return {module: frozenset(names) for module, names in drawing.items()} + + +def _stochastic_assignment_reach() -> dict[str, frozenset[str]]: + """Every spine stage that draws through the keyed-draw helpers, and how. + + A stage draws when its module or a bound helper imports the helpers, or + imports from another first-party module a function that draws through + them. Each such stage maps to the drawing names it imports that way, as + ``module.name`` (empty when it imports the helpers themselves). + """ -def _stages_drawing_through_stochastic_assignment() -> set[str]: from microcosm.build.uk_runtime import graph_kernels - return { - stage + hashed = { + stage: ( + graph_kernels._stage_module(stage), + *graph_kernels._STAGE_HELPER_MODULES.get(stage, ()), + ) for stage in graph_kernels._STAGE_MODULES + } + drawing = _drawing_names( + {module.__name__ for modules in hashed.values() for module in modules} + ) + reach: dict[str, frozenset[str]] = {} + for stage, modules in hashed.items(): if any( "stochastic_assignment" in _imported_module_names(module) - for module in ( - graph_kernels._stage_module(stage), - *graph_kernels._STAGE_HELPER_MODULES.get(stage, ()), - ) - ) - } + for module in modules + ): + reach[stage] = frozenset() + continue + found = set() + for module in modules: + facts = _draw_facts(module.__name__) + if facts is None: + continue + for _, origin, imported in facts.imports: + if imported in drawing.get(origin, ()): + found.add(f"{origin.rpartition('.')[2]}.{imported}") + # A drawing function reached by attribute on an imported module. + for names in facts.references.values(): + for name in names: + head, _, attribute = name.partition(".") + target = facts.module_aliases.get(head) if attribute else None + if target and attribute in drawing.get(target, ()): + found.add(f"{target.rpartition('.')[2]}.{attribute}") + if found: + reach[stage] = frozenset(found) + return reach + + +def _stages_drawing_through_stochastic_assignment() -> set[str]: + return set(_stochastic_assignment_reach()) def test_every_stage_that_draws_through_stochastic_assignment_binds_it() -> None: """``source_hash`` does not follow imports, so every stage whose module or - bound helpers import the keyed-draw helpers binds them, or an edit to how a - uniform is keyed would replay the stage's stale output (microcosm#931), - save the stages ``_STOCHASTIC_ASSIGNMENT_UNBOUND`` names with a reason.""" + bound helpers import the keyed-draw helpers, or import another module's + function that draws through them, binds them, or an edit to how a uniform + is keyed would replay the stage's stale output (microcosm#931), save the + stages ``_STOCHASTIC_ASSIGNMENT_UNBOUND`` names with a reason.""" from microcosm.build import stochastic_assignment from microcosm.build.uk_runtime import graph_kernels - drawing = _stages_drawing_through_stochastic_assignment() + reach = _stochastic_assignment_reach() binding = { stage for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() if stochastic_assignment in helpers } - assert set(_STOCHASTIC_ASSIGNMENT_UNBOUND) <= drawing - assert binding == drawing - set(_STOCHASTIC_ASSIGNMENT_UNBOUND) + assert binding == set(reach) - set(_STOCHASTIC_ASSIGNMENT_UNBOUND) + # The employer pension draw reaches stage 14 through spi_spine alone. + assert reach["frs_hmrc_spine_leaves"] >= { + "employer_pension_contributions.draw_employer_pension_contributions" + } + # An exempt stage imports exactly the drawing names its reason covers. + for stage, (names, _) in _STOCHASTIC_ASSIGNMENT_UNBOUND.items(): + assert reach.get(stage) == names, stage def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( @@ -473,6 +645,8 @@ def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() if stochastic_assignment in helpers ) + # Stage 14 draws through the employer pension helper, which binds nothing. + assert "frs_hmrc_spine_leaves" in stages # The root stage's node is the CREATE kernel. affected = [ graph.node("create_uk_frs" if stage == "frs_spine" else stage).kernel @@ -484,7 +658,8 @@ def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( "cgt_support_split", "hmrc_cgt_asset_type_spine", ) - assert not set(control_stages) & _stages_drawing_through_stochastic_assignment() + drawing = _stages_drawing_through_stochastic_assignment() + assert not set(control_stages) & drawing controls = [graph.node(stage).kernel for stage in control_stages] before = {ref: registry.implementation_hash(ref) for ref in affected + controls} helper_path = Path(stochastic_assignment.__file__).resolve() @@ -497,7 +672,7 @@ def changed_helper_bytes(path: Path) -> bytes: return content monkeypatch.setattr(Path, "read_bytes", changed_helper_bytes) - assert len(affected) == len(_stages_drawing_through_stochastic_assignment()) + assert len(affected) == len(drawing - set(_STOCHASTIC_ASSIGNMENT_UNBOUND)) for ref in affected: assert registry.implementation_hash(ref) != before[ref], ref for ref in controls: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py index 7fe8207f06..ec092078c8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py @@ -207,6 +207,26 @@ def test_copies_of_spi_copies_take_their_parents_ids_plus_k_times_the_radix() -> # Some copies are of SPI copies, and some of base households. assert (copied["marker"] > 1_000).any() assert (copied["marker"] < 1_000).any() + # The identity tool's old-spine gate reads the stage's copies as on the + # table (microcosm#931). + assert _identity_tool()._copy_id_rows_off_the_declared_table(household) == 0 + + +def _identity_tool(): + import importlib.util + + from test_support.paths import paths_for + + path = ( + paths_for("microcosm-build").repository + / "tools" + / "verify_uk_identity_stability.py" + ) + spec = importlib.util.spec_from_file_location("verify_uk_identity_stability", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module def test_the_split_refuses_incoming_ids_at_its_copy_id_radix() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index aac803ed75..e3d9d97ae6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -892,14 +892,28 @@ def test_dropping_the_split_layer_under_a_kept_clone_layer_is_refused( def test_a_copy_without_a_root_fails_closed(self) -> None: tool = _load_tool() frame = _hand_split_frame(cloned=False) + refusal = ( + "CGT support copy(ies) without a root household at the declared radix " + "of the copy-id table's 'cgt_support_split' field." + ) household = frame.table("household").copy() # Flagging id 2 as a copy gives it ordinal 0 in the cgt_support_split # field: no root. household.loc[household["household_id"] == 2, HOUSEHOLD_IS_CGT_SUPPORT_COPY] = ( True ) - with pytest.raises(ValueError, match="without a root household"): + with pytest.raises(ValueError) as caught: + tool._pre_split_household_weights(_rebuild(frame, household=household)) + assert str(caught.value) == f"1 {refusal}" + # Flagging root 1 instead leaves its two copies' parent in the frame + # but outside the rows the split copied: all three are orphans. + household = frame.table("household").copy() + household.loc[household["household_id"] == 1, HOUSEHOLD_IS_CGT_SUPPORT_COPY] = ( + True + ) + with pytest.raises(ValueError) as caught: tool._pre_split_household_weights(_rebuild(frame, household=household)) + assert str(caught.value) == f"3 {refusal}" def test_the_stored_copy_index_is_read_and_held_to_the_id_scheme(self) -> None: """The explicit index (microcosm#1045 review) must agree with the ids.""" @@ -922,8 +936,14 @@ def test_the_stored_copy_index_is_read_and_held_to_the_id_scheme(self) -> None: tampered.loc[ tampered["household_id"] == copy_id, CGT_SUPPORT_COPY_INDEX_COLUMN ] = 7 - with pytest.raises(ValueError, match="disagrees with the id scheme"): + with pytest.raises(ValueError) as caught: tool._pre_split_household_weights(_rebuild(frame, household=tampered)) + assert str(caught.value) == ( + "CGT support copy index stored on 1 copy(ies) disagrees with the id " + "scheme (the id's ordinal in the copy-id table's 'cgt_support_split' " + "field)." + ) + assert str(copy_id) not in str(caught.value) unflagged = household.copy() unflagged.loc[ @@ -1109,6 +1129,35 @@ def test_a_copy_whose_lineage_disagrees_with_its_id_is_refused(self) -> None: household.loc[twin, HOUSEHOLD_IS_CGT_CLONE] = False assert tool._copy_id_rows_off_the_declared_table(household) == 1 + def test_the_gate_holds_each_stored_ordinal_to_the_ids_exactly(self) -> None: + # The SPI channel's ordinal (copy 1, band donor 2) and the high-value + # copy's are checked against the id only here; a nonzero-marker reading + # would pass a band donor recorded as the SPI copy. + from microcosm.build.uk_runtime.household_lineage import ( + HIGH_VALUE_SUPPORT_INDEX_COLUMN, + ) + + tool = _load_tool() + household = _lineage_household(_chain("table").household) + for column, stored, tampered_to in ( + ("household_support_clone_index", 2, 1), + (HIGH_VALUE_SUPPORT_INDEX_COLUMN, 2, 3), + ): + tampered = household.copy() + row = int(np.flatnonzero(tampered[column].to_numpy() == stored)[0]) + tampered.loc[row, column] = tampered_to + assert tool._copy_id_rows_off_the_declared_table(tampered) == 1, column + + def test_the_gate_expects_nothing_in_the_vintage_field(self) -> None: + # Nothing mints a vintage ordinal yet, so a base id moved into that + # field is off the table although no lineage column names the field. + tool = _load_tool() + moved = _lineage_household(_chain("table").household) + moved.loc[0, "household_id"] = int( + moved.loc[0, "household_id"] + ) + copy_id_table.radix("vintage") + assert tool._copy_id_rows_off_the_declared_table(moved) == 1 + def test_the_tool_refuses_a_spine_built_before_the_table( self, monkeypatch, tmp_path ) -> None: @@ -1221,3 +1270,155 @@ def test_a_support_copy_decodes_exactly_above_2_53(self) -> None: floated = household.assign(household_id=household["household_id"].astype(float)) with pytest.raises(TypeError, match="integer household ids"): tool._support_copy_lineage(floated) + + +# --- the receipts on frames the CGT stages built (microcosm#931) ----------------- +# +# The split scenario above run through the support split (42), the incidence +# twin (43), a redraw by hand (44: each twin's carrier takes a liable gain), +# the anchor (45) and the residential split (46), with Table 8 totals sized to +# the frame's liable mass. Every value is synthetic. + + +def _residentially_split(): + """The split scenario after stages 42 to 46, and the Table 8 facts it used.""" + + import dataclasses + + from microcosm.build.uk_runtime.cgt_asset_type import ( + CGT_STOCK_HOUSEHOLD_COLUMNS, + CGT_STOCK_PERSON_COLUMNS, + load_hmrc_cgt_asset_type_facts, + ) + from microcosm.build.uk_runtime.cgt_residential_split import ( + split_cgt_residential_households, + ) + from microcosm.build.uk_runtime.cgt_structure import anchor_cgt_incidence + from microcosm.build.uk_runtime.national_frame import uk_household_weight_kind + from test_support.microcosm_build.uk_cgt_support import support_tables + + person, benunit, household, weights = support_tables( + weights=_SPLIT_WEIGHTS, wealth=_SPLIT_WEALTH, incomes=_SPLIT_INCOMES + ) + # The residential split reads the asset stocks; those the scenario does + # not carry are zero. + for column in CGT_STOCK_PERSON_COLUMNS: + person[column] = person.get(column, 0.0) + for column in CGT_STOCK_HOUSEHOLD_COLUMNS: + household[column] = household.get(column, 0.0) + split = split_cgt_support_households( + uk_national_frame( + person=person, + benunit=benunit, + household=household, + household_weights=weights, + time_period="2024", + ), + distribution=support_distribution(_SPLIT_TOP_CELLS), + parameters=PARAMETERS, + ) + twinned = clone_cgt_incidence( + split.frame, distribution=load_advani_summers_distribution() + ).frame + redrawn_person = twinned.table("person").copy() + carrier = redrawn_person["capital_gains"].to_numpy() != 0.0 + redrawn_person["capital_gains"] = np.where( + carrier, 20_000.0 + 1_000.0 * (redrawn_person["person_id"] % 7), 0.0 + ) + anchored = anchor_cgt_incidence( + uk_national_frame( + person=redrawn_person, + benunit=twinned.table("benunit"), + household=twinned.table("household"), + time_period="2024", + weight_kind=uk_household_weight_kind(twinned), + household_weights=twinned.weights_for("household").values, + mass_log=twinned.mass_log, + ), + distribution=load_advani_summers_distribution(), + parameters=PARAMETERS, + ).frame + anchored_person = anchored.table("person") + by_household = pd.Series( + anchored.weights_for("household").values, + index=anchored.table("household")["household_id"], + ) + person_weight = anchored_person["person_household_id"].map(by_household) + gains = anchored_person["capital_gains"] + liable = gains > PARAMETERS.annual_exempt_amount + count = 0.3 * float(person_weight[liable].sum()) + total = 0.3 * float((person_weight[liable] * gains[liable]).sum()) + facts = dataclasses.replace( + load_hmrc_cgt_asset_type_facts(), + table8a_taxpayers_total=count, + table8a_gains_total=total, + table8b_individuals_taxpayers=count, + table8b_individuals_gains=total, + table8b_all_taxpayers=count, + table8b_all_gains=total, + ) + result = split_cgt_residential_households( + anchored, facts=facts, parameters=PARAMETERS + ) + return result.frame, facts + + +class TestReceiptsOnFramesTheStagesBuilt: + def test_the_old_spine_gate_passes_every_layer_the_stages_minted(self) -> None: + tool = _load_tool() + _, twinned = _split_then_cloned() + with_arms, _ = _residentially_split() + for frame in (twinned, with_arms): + assert ( + tool._copy_id_rows_off_the_declared_table(frame.table("household")) == 0 + ) + + def test_the_arms_the_stage_built_decode_and_receipt_green(self) -> None: + from microcosm.build.uk_runtime.cgt_residential_split import ( + HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, + ) + + tool = _load_tool() + frame, facts = _residentially_split() + problems: dict[str, object] = {} + residential = tool._e8_residential_split( + frame, problems, parameters=PARAMETERS, facts=facts, permutation_seed=7 + ) + assert problems == {} + radix = copy_id_table.radix("cgt_residential_split") + assert residential["id_multiplier"] == radix == 10**18 + # One arm for each of the 11 twins (six households and five copies), + # whose carrier alone gains. + assert residential["arms"] == residential["households_split"] == 11 + # Every arm's people are its source's people shifted by the arm's + # index times the radix: the decode found the row the stage copied. + household = frame.table("household") + lineage = tool._residential_arm_lineage(household) + ids = household["household_id"].to_numpy() + assert int(ids[lineage.arm_positions].min()) > 2**53 + members = ( + frame.table("person") + .groupby("person_household_id")["person_id"] + .apply(lambda people: sorted(int(value) for value in people)) + ) + for arm, source, index in zip( + lineage.arm_positions, + lineage.source_positions, + lineage.arm_index, + strict=True, + ): + shifted = [value - int(index) * radix for value in members[ids[arm]]] + assert shifted == members[ids[source]] + # The layers below, with the arms folded away, receipt green too. + below = tool._drop_stacked_layers(frame, [HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE]) + pairs = tool._e8_clone_pairs(below, problems, parameters=PARAMETERS) + support = tool._e8_support_split( + below, + problems, + distribution=support_distribution(_SPLIT_TOP_CELLS), + parameters=PARAMETERS, + permutation_seed=7, + ) + assert problems == {} + assert pairs["anchored"] is True and pairs["pairs"] == 11 + assert support["copies"] == 5 and support["families"] == 3 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py index 48a940208f..565b9bc999 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py @@ -491,10 +491,50 @@ def test_spine_sampling_fractional_source_fails_closed() -> None: def test_spine_sampling_orphan_derivative_fails_closed() -> None: - with pytest.raises(ValueError, match="999"): + # The refusals count families and name no source household (microcosm#931). + with pytest.raises(ValueError) as caught: uk_spine_source_family_units(_spine_family_frame(orphan_source=999)) + assert str(caught.value) == ( + "UK spine sample: 1 source_household_id family key(s) lack a " + "household_id-bearing raw row." + ) def test_spine_sampling_multi_region_raw_family_fails_closed() -> None: - with pytest.raises(ValueError, match="spans more than one raw-row region.*1"): + with pytest.raises(ValueError) as caught: uk_spine_source_family_units(_spine_family_frame(multi_region_family=True)) + assert str(caught.value) == ( + "UK spine sample: 1 source family(ies) span more than one raw-row region." + ) + + +def test_spine_sampling_counts_the_families_a_sample_left_incomplete( + monkeypatch, +) -> None: + from microcosm.build.uk_runtime import national_sampling + + real = national_sampling.uk_spine_source_family_units + calls: list[int] = [] + + def relabel_one_row_after_sampling(frame): + units, strata = real(frame) + calls.append(len(units)) + if len(calls) == 2: + # The first sampled row, a family of one, joins the next sampled + # family, which then holds a row more than in the full frame. + units = units.copy() + units[0] = next(unit for unit in units if unit != units[0]) + return units, strata + + monkeypatch.setattr( + national_sampling, + "uk_spine_source_family_units", + relabel_one_row_after_sampling, + ) + with pytest.raises(ValueError) as caught: + sample_uk_spine_frame( + _spine_family_frame(families_per_region=100), fraction=0.5, seed=19 + ) + assert str(caught.value) == ( + "UK spine sample retained 1 incomplete source family(ies)." + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dry_run.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dry_run.py index 7693029547..ddf92699b5 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dry_run.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_dry_run.py @@ -507,8 +507,62 @@ def test_driver_dry_run_rejects_broken_links(monkeypatch, tmp_path) -> None: monkeypatch.setattr( sys, "argv", _dry_run_argv(input_h5, output_dir, crosswalk_path) ) - with pytest.raises(ValueError, match="person_household_id"): + with pytest.raises(ValueError, match="person_household_id") as caught: builder.main() + # The plan refuses as the build does: by count, naming no id (microcosm#931). + assert str(caught.value) == ( + "person.person_household_id names a household absent from household on " + "1 person row(s)." + ) + + +@pytest.mark.parametrize( + ("table", "column", "value", "refusal"), + [ + ( + "household", + "household_id", + 2, + "household.household_id must be unique; 1 row(s) repeat an earlier " + "row's value.", + ), + ( + "person", + "person_benunit_id", + 848_484, + "person.person_benunit_id names a benunit absent from benunit on 1 " + "person row(s).", + ), + ], +) +def test_driver_dry_run_input_refusals_count_rows_and_name_no_id( + monkeypatch, tmp_path, table, column, value, refusal +) -> None: + pytest.importorskip("tables") + pytest.importorskip("h5py") + builder = _load_builder_module() + frames = { + "household": _household_frame(), + "person": _person_frame(), + "benunit": _benunit_frame(), + } + # Household 1 repeats household 2's id, or person 1001 names a benefit + # unit the input does not hold. + frames[table].loc[0, column] = value + input_h5 = tmp_path / "populace_uk_2023.h5" + _write_toy_h5(input_h5, **frames) + crosswalk_path = tmp_path / "crosswalk.csv.gz" + _crosswalk_frame().to_csv(crosswalk_path, index=False) + output_dir = tmp_path / "out" + + monkeypatch.setattr( + sys, "argv", _dry_run_argv(input_h5, output_dir, crosswalk_path) + ) + with pytest.raises(ValueError) as caught: + builder.main() + assert str(caught.value) == refusal + assert str(value) not in str(caught.value) + assert not (output_dir / builder.DRY_RUN_PLAN_FILENAME).exists() def test_driver_full_build_records_weight_chain_and_lineage( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_size_evaluation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_size_evaluation.py index cef43e176e..d02616bf86 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_size_evaluation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_size_evaluation.py @@ -748,6 +748,98 @@ def test_a_file_that_keeps_its_sources_still_decodes_its_spine_stretch( assert result["stretch"]["vs_pool_design"]["max"] == pytest.approx(1.0) +def test_an_older_dense_run_above_2_53_decodes_exactly_at_every_basis( + tmp_path: Path, +) -> None: + # A spine minted by the retired rule reaches 16 digits and its dense + # clones 18 at K=15. Odd ids above 2**53 have no float64 twin, so a + # subtraction through float would land off the spine ids. + spine_ids = np.array([2**53 + 1, 2**53 + 3, 2**53 + 5, 10**16 - 1], dtype=np.int64) + assert all(float(value) != value for value in spine_ids.tolist()) + clones, multiplier = 15, 10**16 + clone_index = np.repeat(np.arange(clones, dtype=np.int64), len(spine_ids)) + dense_ids = np.tile(spine_ids, clones) + clone_index * multiplier + weights = np.array([10.0, 20.0, 30.0, 40.0]) + run = load_run( + _with_households( + _write_run_dir(tmp_path, "dense", size_run=False), + pd.DataFrame( + { + "household_id": dense_ids, + "household_weight": np.tile(weights, clones) / clones, + "clone_index": clone_index, + } + ), + ), + label="dense", + ) + spine = pd.DataFrame({"household_id": spine_ids, "household_weight": weights}) + for kwargs, basis in ( + ( + {"id_multiplier": multiplier, "spine_weights": spine, "n_clones": clones}, + "recorded", + ), + ({"spine_weights": spine, "n_clones": clones}, "spine"), + ({}, "inferred_from_clone_zero_ids"), + ): + result = weight_tables(run, **kwargs) + assert result["source_id_multiplier"] == multiplier + assert result["source_id_multiplier_basis"] == basis + assert result["distinct_source_households"] == len(spine_ids) + if kwargs: + # Every clone carries a fifteenth of its spine household: exact. + assert result["stretch"]["vs_pool_design"]["max"] == pytest.approx(1.0) + + +def test_spine_weights_without_a_dense_rows_household_are_refused_by_count( + tmp_path: Path, +) -> None: + import traceback + + ids = np.array([1, 2, 3, 424_242], dtype=np.int64) + run = load_run( + _with_households( + _write_run_dir(tmp_path, "dense", size_run=False), + pd.DataFrame( + { + "household_id": ids, + "household_weight": np.full(len(ids), 5.0), + "clone_index": np.zeros(len(ids), dtype=np.int64), + } + ), + ), + label="dense", + ) + spine = pd.DataFrame({"household_id": ids[:3], "household_weight": [5.0] * 3}) + with pytest.raises(ValueError) as caught: + weight_tables(run, spine_weights=spine, n_clones=1, id_multiplier=10**6) + assert str(caught.value) == ( + "spine weights do not cover the spine household of 1 dense row(s)" + ) + # Counted before the lookup: no pandas KeyError listing the id is chained. + assert caught.value.__cause__ is None and caught.value.__context__ is None + assert "424242" not in "".join(traceback.format_exception(caught.value)) + + +def test_a_negative_clone_index_is_refused_by_count(tmp_path: Path) -> None: + run = load_run( + _with_households( + _write_run_dir(tmp_path, "dense", size_run=False), + pd.DataFrame( + { + "household_id": np.array([1, 2, 535_353], dtype=np.int64), + "household_weight": np.full(3, 5.0), + "clone_index": np.array([0, 0, -1], dtype=np.int64), + } + ), + ), + label="dense", + ) + with pytest.raises(ValueError) as caught: + weight_tables(run) + assert str(caught.value) == "1 dense row(s) carry a negative clone_index" + + def test_area_support_and_gate_tables(tmp_path: Path) -> None: run = load_run( _write_run_dir(tmp_path, "candidate", size_run=True), label="candidate" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index 822af31afd..300e3fd307 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -406,6 +406,10 @@ def test_spi_copies_and_donors_take_their_base_roots_ids_plus_the_radix() -> Non .eq(SPI_INCOME_BAND_DONOR_CLONE_INDEX) .all() ) + # The identity tool's old-spine gate reads both layers as on the table. + gate = _identity_tool()._copy_id_rows_off_the_declared_table + assert gate(frame.table("household")) == 0 + assert gate(result.frame.table("household")) == 0 def test_stack_scales_seats_then_mass_on_a_frame_too_small_for_the_plan() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py index 085e61203e..29b1b8c9e3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py @@ -789,8 +789,9 @@ def test_spi_fill_rejects_missing_source_ids() -> None: entity="person", columns=["employment_income"], ) - assert "1 row(s) repeat" in str(caught.value) - assert "1001" not in str(caught.value) + assert str(caught.value) == ( + "donor.person_id must be unique; 1 row(s) repeat an earlier row's id." + ) def test_spi_support_input_refusals_count_rows_and_name_no_id() -> None: @@ -818,7 +819,9 @@ def test_spi_support_input_refusals_count_rows_and_name_no_id() -> None: household=household, spi_household_count=None, ) - assert "1 row(s) repeat" in str(caught.value) + assert str(caught.value) == ( + "household.household_id must be unique; 1 row(s) repeat an earlier row's value." + ) with pytest.raises(ValueError, match="absent from household") as caught: create_uk_spi_support_tables( diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 13fa9c85d2..0a60c5db29 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -73,6 +73,8 @@ from microcosm.build.uk_runtime.rowwise_dataset import ( UK_SPINE_LINEAGE_COLUMNS, _refuse_preassigned_geography, + _require_person_links, + _require_unique, refuse_dense_clone_id_overflow, ) from microcosm.frame import engine_tables @@ -1462,38 +1464,18 @@ def _validate_dry_run_input( benunit_ids: pd.DataFrame, ) -> None: """Mirror the real build's input refusals so a plan cannot bless an - input the build would reject.""" + input the build would reject. + + The build's own checks run here, so the plan's refusals are the build's: + they print counts, never an id (microcosm#931). + """ if input_h5.suffix != ".h5": raise ValueError("UK single-year dataset path must end with '.h5'.") - for frame, column, label in ( - (household, "household_id", "household"), - (person_ids, "person_id", "person"), - (benunit_ids, "benunit_id", "benunit"), - ): - if frame[column].isna().any(): - raise ValueError(f"{label}.{column} contains missing values.") - if frame[column].duplicated().any(): - duplicates = frame.loc[frame[column].duplicated(), column].unique() - raise ValueError( - f"{label}.{column} must be unique; duplicate value(s): " - f"{list(map(str, duplicates[:5]))}." - ) - household_ids = set(household["household_id"]) - missing_households = sorted(set(person_ids["person_household_id"]) - household_ids) - if missing_households: - raise ValueError( - "person.person_household_id contains value(s) absent from " - f"household: {missing_households[:5]}." - ) - missing_benunits = sorted( - set(person_ids["person_benunit_id"]) - set(benunit_ids["benunit_id"]) - ) - if missing_benunits: - raise ValueError( - "person.person_benunit_id contains value(s) absent from benunit: " - f"{missing_benunits[:5]}." - ) + _require_unique(household, "household_id", label="household") + _require_unique(person_ids, "person_id", label="person") + _require_unique(benunit_ids, "benunit_id", label="benunit") + _require_person_links(person_ids, benunit_ids, household) def _realized_area_support( diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index b3b94c1edf..136437de7b 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -2159,13 +2159,19 @@ def _pre_residential_household_weights(frame) -> np.ndarray: # # Every UK copy layer mints its copies' ids in a decimal field of its own, # declared in ``uk_runtime.copy_id_table``: copy ``k`` of a row in field ``f`` -# carries the row's ids plus ``k`` times ``f``'s radix. The receipts decode a -# copy id only through ``_copy_ordinals_and_parents``. A spine built before -# the table offset each layer's copies by a power of ten above the largest id -# the layer saw; ``_require_copy_ids_on_the_declared_table`` refuses such a -# spine, which is receipted with the tool at the commit that built it. -# Decoding that retired rule as well would mean changing these two functions, -# and nothing else. +# carries the row's ids plus ``k`` times ``f``'s radix. The receipts decode +# the CGT support copies and the residential arms through +# ``_copy_ordinals_and_parents``, and pair the incidence twins through +# ``cgt_structure.pair_clone_households``, which pairs at the twin field's +# radix too. A spine built before the table offset each layer's copies by a +# power of ten above the largest of the five id columns of the rows the layer +# saw; ``_require_copy_ids_on_the_declared_table`` refuses such a spine, which +# is receipted with the tool at the commit that built it. The refusal is one +# small function. Decoding the retired rule as well would take more: the +# person and benefit-unit tables threaded back through +# ``_support_copy_lineage`` and ``_residential_arm_lineage`` (which read the +# household table only) and their four callers, and a tool-local twin pairing +# at the three ``pair_clone_households`` call sites. #: The old-spine refusal, after its count of household rows. _OFF_TABLE_REFUSAL = ( From 8ec385c14c947ca285fa99a5ab837a54b6dd6c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:08:12 +0000 Subject: [PATCH 017/108] Count CGT support copies on the source-scaled weight (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CGT support split divided each selected household into ceil(w / 60) copies of its own weight. Once (b) clones households at the spine's start, each geographic clone carries 1/K of its source's weight, so a clone's count would shrink with K and the K=1 spine would stop nesting in the K=15 one. As María ruled on 7 Nov, the count now reads the geographic clone count times the row's own weight, so every clone of a household gets the count its source gets at K=1. Nothing moves at K=1. - Count: cgt_support_copy_counts(weights, maximum_copy_weight, *, clone_count) returns max(1, ceil(K x w / 60)). For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so an exact multiple of 60 keeps its count at any K (at K=7, 7 x (60m / 7) lands above 60m for 626 of m < 20,000 in a synthetic sweep). The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K=1. At K=1 nothing is snapped, and the count is ceil(w / 60) bit for bit. - Stage: split_cgt_support_households takes a required clone_count, and the transform reads it from the frame (geographic_clone_count): one more than the largest household_clone_index, or 1 without the column. A non-integer column, a missing or negative index, and indices that are not exactly 0 to K - 1 are refused by count. Copies keep w / n, so each weighs at most 60 / K. The selection walk still accumulates each row's own weight, and nothing sums geographic siblings. - Receipt and gate: the receipt's parameters gain clone_count and nothing else. stage_health compares K x the heaviest copy and K x the heaviest selected household with 60, so a receipt at K > 1 no longer fails falsely. At K=1 both checks and their messages are byte-identical, and a receipt without a positive integer clone_count is refused. The identity tool's recompute reads K from the pre-split table and passes it on; its receipt is unchanged. - Texts: copy_rule in the code dict, sources.yaml and source_stages.json now reads 'copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies'. The stage notes give the cap as 60 divided by the geographic clone count before the incidence clone. The docstrings, the cap's comment and a changelog fragment are updated to match. The mass-change reason is unchanged. What moves: no value at K=1. On synthetic frames the pre-commit and new code give the same results: the stage's ids, copy counts, weights (bit for bit), mass records and frame content identity, the identity tool's receipts, and the gate's verdicts and messages. The only difference is that stage evidence gains clone_count. Implementation hashes move for cgt_support_split, and for high_value_home_support, which binds cgt_support. So node keys move from stage 26 on. The spine gates hash stage_health, and every stage after the assembled gate waits on its verdict, so in a gated build every node from stage 14 on re-keys. cgt_support_split's stage contract moves with its texts. Regenerated: - source_stages.json (copy_rule and notes, re-derived from the typed projection); - the coverage manifest's 25 source digests; - the H2 fixture: the two declaration lines in fixture.json, and the stage's contract hash in uk_spine.json. Tests: - K=1 counts equal ceil(w / 60) bitwise on random and edge weights: exact multiples of 60, one ulp either side, tiny and large. - K=3, 7 and 15 replicas count as their sources, including the K=7 exact multiples the unsnapped rule overcounts. The snap's documented trade is pinned. - The clone-count refusals ({0, 2}, negative, missing, float, bool) give counts. - On K=3 and K=7 clone-first frames, the transform gives every clone its source's copies, with weights nested within rtol 1e-12 and every copy at most 60 / K. - The packaged gate passes a K=3 stage receipt with w <= 60 < 3w, and fails it when the count is ignored. Hand-built receipts pin the K=1 messages and both scaled checks. - The identity tool's K=1 receipt equals the one ceil(w / 60) gives, with no new key. At K=3 it receipts green, and it reports a recompute that ignores K. Eight source mutations each fail a new test. The 51 touched engine-free files pass (2,317 tests). The coverage manifest's check passes, the H2 fixture regenerates and passes its check, and H2 parity passes in the engine slot. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 5 + .../uk/release_input_coverage_manifest.json | 50 +- .../src/microcosm/build/uk/source_stages.json | 4 +- .../src/microcosm/build/uk/spec/sources.yaml | 4 +- .../microcosm/build/uk_runtime/cgt_support.py | 163 ++++++- .../build/uk_runtime/stage_health.py | 29 +- .../engine_free/uk/test_uk_cgt_support.py | 444 +++++++++++++++++- .../uk/test_uk_identity_stability_receipts.py | 122 +++++ .../engine_free/uk/test_uk_stage_health.py | 83 ++++ .../parity/uk_spine/sources/fixture.json | 4 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- .../microcosm_build/uk_clone_first_frames.py | 60 +++ tools/verify_uk_identity_stability.py | 20 +- 13 files changed, 921 insertions(+), 69 deletions(-) create mode 100644 changelog.d/931-cgt-copies-source-weight.changed.md diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md new file mode 100644 index 0000000000..ea7ccebb28 --- /dev/null +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -0,0 +1,5 @@ +The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so every clone of a household gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. + +- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt is unchanged. +- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies". The stage notes give the cap as 60 divided by the geographic clone count before the incidence clone. +- **What moves.** No value moves at K = 1. On synthetic frames, the stage's output and the identity tool's receipt are unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The stage evidence gains `clone_count`. `cgt_support_split`'s stage contract moves with its texts. Its implementation hash moves, and so does `high_value_home_support`'s, which binds `cgt_support`. The spine gates hash `stage_health`, so their keys move, and with them every spine stage after the assembled gate. The coverage manifest's source digests and the H2 fixture's declarations and contract hash are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 7a15e60db8..e7a7914edc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "ad8e27ea5f0534ada3a68390f4e85849f000a8f5c14cd0ae381933b782457df2", + "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index e669ee5f94..e8a1b802d4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -7398,7 +7398,7 @@ ], "selection": "within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "maximum_copy_weight": 60.0, - "copy_rule": "copies = ceil(household weight / maximum_copy_weight); every copy carries weight / copies", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies", "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "flag_column": "household_is_cgt_support_copy", "copies_column": "cgt_support_copies", @@ -7416,7 +7416,7 @@ "cgt_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 before the clone (30 after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." }, { "stage": "cgt_incidence_clone", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index c43258c8ca..9d3f8e831d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -6011,7 +6011,7 @@ stages: - non_residential_property_value selection: within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused maximum_copy_weight: 60.0 - copy_rule: copies = ceil(household weight / maximum_copy_weight); every copy carries weight / copies + copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies id_remapping: copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused flag_column: household_is_cgt_support_copy copies_column: cgt_support_copies @@ -6026,7 +6026,7 @@ stages: - cgt_support_copies - cgt_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 before the clone (30 after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' + notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' - stage: cgt_incidence_clone survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index 02cbce14fc..a298a0a96f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -10,8 +10,9 @@ descending investable wealth until their weight covers the band's support mass (the published count of gainers at or above GBP 250,000 in that income column, doubled for the clone split and scaled by the declared headroom of -1.25) and divides every selected household into ``ceil(weight / 60)`` -identical copies at equal weight, each carrying its index ``k`` in +1.25) and divides every selected household into ``ceil(K x weight / 60)`` +identical copies at equal weight, ``K`` the frame's geographic clone count +(1 on a national spine; microcosm#931), each carrying its index ``k`` in ``cgt_support_copy_index`` (0 on roots and unsplit households), the explicit lineage the geography identity kernel keys on. No value moves, no draw is made, and every household's mass and the total household mass are conserved @@ -85,8 +86,9 @@ #: Covers clones whose prior draws at or below zero and the quarter of the #: redraw's rank key the prior carries. CGT_SUPPORT_HEADROOM = 1.25 -#: A copy weighs at most this before the clone (half of it after), so the -#: GBP 5m+ band seats at least a hundred rows. +#: A copy weighs at most this divided by the geographic clone count before +#: the incidence clone (half of that after): at most 60 on a national spine, +#: so the GBP 5m+ band seats at least a hundred rows. CGT_SUPPORT_MAXIMUM_COPY_WEIGHT = 60.0 #: The copy-id field copy ``k`` takes ``k`` in: ordinals 1 to 9,999, so a #: household is divided into at most 10,000 rows, root included @@ -120,6 +122,12 @@ "household's mass and the total household mass are conserved." ) _HOUSEHOLD_SUPPORT_CHANNEL_COLUMN = "household_support_channel" +#: The geographic clone index the clone-first spine gives every household +#: (``rowwise_dataset.ladder_clone_index_column("household")``, the +#: ``geographic_support`` branch of the UK copy lineage; microcosm#931). It is +#: spelled here, not imported, so a change to it re-keys this stage; a test +#: holds the two spellings equal. +_GEOGRAPHIC_CLONE_INDEX_COLUMN = "household_clone_index" _PERSON_ID_COLUMNS = ("person_id", "person_household_id", "person_benunit_id") _POLICY_PARAMETER_NAMES = ( "gov.hmrc.income_tax.allowances.personal_allowance.amount", @@ -134,8 +142,9 @@ class UKCGTSupportSplitResult: ``band_rows`` carry one mapping per Table 3 income band, in the order of :data:`CGT_SUPPORT_INCOME_BAND_LOWER_BOUNDS`; the totals, the mass pair, - the three parameters and the support-channel split of the selected - households make up the rest of :meth:`evidence`. + the four parameters (the geographic clone count the copies were counted + on among them) and the support-channel split of the selected households + make up the rest of :meth:`evidence`. """ frame: Frame @@ -153,6 +162,7 @@ class UKCGTSupportSplitResult: new_total: float frs_selected: int spi_selected: int + clone_count: int clone_split_factor: int = CGT_SUPPORT_CLONE_SPLIT_FACTOR headroom: float = CGT_SUPPORT_HEADROOM maximum_copy_weight: float = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT @@ -182,6 +192,7 @@ def evidence(self) -> dict[str, object]: "clone_split_factor": int(self.clone_split_factor), "headroom": float(self.headroom), "maximum_copy_weight": float(self.maximum_copy_weight), + "clone_count": int(self.clone_count), }, "support_channel_split": { "frs": int(self.frs_selected), @@ -197,8 +208,10 @@ class UKCGTSupportSplitStageTransform: ``distribution`` defaults to the vendored Table 3 joint; ``parameters`` default to the policyengine-uk allowance parameters at the frame's build period, read at apply time so the base package never imports the engine - at import time. Constructing the transform binds the manifest to the - reviewed operation dictionary and the vendored pins. + at import time. The geographic clone count the copies are counted on is + read from the frame (:func:`geographic_clone_count`). Constructing the + transform binds the manifest to the reviewed operation dictionary and the + vendored pins. """ stage: SourceStageSpec @@ -218,7 +231,10 @@ def __call__(self, frame: Frame) -> Frame: if parameters is None: parameters = uk_cgt_policy_parameters(uk_time_period(frame)) result = split_cgt_support_households( - frame, distribution=distribution, parameters=parameters + frame, + distribution=distribution, + parameters=parameters, + clone_count=geographic_clone_count(frame.table("household")), ) object.__setattr__(self, "last_result", result) return result.frame @@ -418,12 +434,94 @@ def select_cgt_support_households( return selected, tuple(rows) +def _require_clone_count(clone_count: object) -> int: + """``clone_count`` as an int, refused unless it is a positive integer.""" + + if ( + isinstance(clone_count, bool) + or not isinstance(clone_count, int | np.integer) + or clone_count < 1 + ): + raise ValueError( + "The CGT support split needs a positive integer geographic clone " + f"count, got {clone_count!r}." + ) + return int(clone_count) + + +def geographic_clone_count(household: pd.DataFrame) -> int: + """The frame's geographic clone count ``K``: one more than its largest index. + + The clone-first spine (microcosm#931) gives every household the index of + its geographic clone in ``household_clone_index``, 0 to ``K - 1``; a + household table without that column, as on every national spine today, + is one clone. Refuses a non-integer index column, a household without an + index, a negative index and indices that are not exactly 0 to ``K - 1`` + (an index below the largest that no household carries). Each refusal + gives a count, never an id. + """ + + column = _GEOGRAPHIC_CLONE_INDEX_COLUMN + if column not in household.columns: + return 1 + values = household[column] + if pd.api.types.is_bool_dtype(values.dtype) or not ( + pd.api.types.is_integer_dtype(values.dtype) + ): + raise ValueError( + "The CGT support split counts geographic clones on an integer " + f"{column!r} column; it has dtype {values.dtype}." + ) + rule = ( + f"the CGT support split counts K geographic clones as the {column!r} " + "values 0 to K - 1 (microcosm#931)." + ) + unindexed = int(values.isna().sum()) + if unindexed: + raise ValueError(f"{unindexed} household(s) carry no {column!r}; {rule}") + indices = values.to_numpy(dtype=np.int64) + if not indices.size: + return 1 + negative = int((indices < 0).sum()) + if negative: + raise ValueError(f"{negative} household(s) carry a negative {column!r}; {rule}") + clone_count = int(indices.max()) + 1 + absent = clone_count - int(np.unique(indices).size) + if absent: + raise ValueError( + f"{absent} of the {clone_count} indices from 0 to the largest are " + f"carried by no household; {rule}" + ) + return clone_count + + def cgt_support_copy_counts( weights: Sequence[float] | np.ndarray, maximum_copy_weight: float = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + *, + clone_count: int, ) -> np.ndarray: - """``ceil(weight / maximum_copy_weight)`` per household, at least one.""" + """``max(1, ceil(clone_count x weight / maximum_copy_weight))`` per household. + + ``clone_count`` is the frame's geographic clone count ``K`` + (:func:`geographic_clone_count`; 1 on a national spine). A geographic + clone carries ``1 / K`` of its source household's weight, so ``K x w`` is + its weight scaled back to the source: every clone of a household gets the + count its source gets at ``K = 1``, and each of its copies weighs at most + ``maximum_copy_weight / K`` (microcosm#931). + + For ``K > 1`` the quotient is snapped to the nearest integer when it lies + within ``K`` ulps of it. A clone's weight ``W / K`` is rounded, so when + ``W`` is an exact multiple of ``maximum_copy_weight`` the product + ``K x (W / K)`` can land an ulp above the multiple, and ``ceil`` would add + a copy that ``K = 1`` does not make (at ``K = 7``, for one). The snap + gives up the converse case: a quotient within ``K`` ulps above an integer, + which ``K = 1`` rounds up, counts one copy fewer. At ``K = 1`` no weight + was divided, so nothing is snapped and the count is ``ceil(w / + maximum_copy_weight)`` bit for bit. + """ + clone_count = _require_clone_count(clone_count) if not np.isfinite(maximum_copy_weight) or maximum_copy_weight <= 0.0: raise ValueError( f"CGT support maximum copy weight must be positive, got " @@ -432,7 +530,17 @@ def cgt_support_copy_counts( values = np.asarray(weights, dtype=float) if not np.isfinite(values).all() or (values < 0.0).any(): raise ValueError("CGT support copy counts need finite, non-negative weights.") - counts = np.ceil(values / float(maximum_copy_weight)).astype("int64") + if clone_count == 1: + quotient = values / float(maximum_copy_weight) + else: + quotient = (clone_count * values) / float(maximum_copy_weight) + nearest = np.rint(quotient) + quotient = np.where( + np.abs(quotient - nearest) <= clone_count * np.spacing(nearest), + nearest, + quotient, + ) + counts = np.ceil(quotient).astype("int64") return np.maximum(counts, 1) @@ -454,24 +562,30 @@ def split_cgt_support_households( *, distribution: HMRCCapitalGainsJointDistribution, parameters: UKCGTPolicyParameters, + clone_count: int, headroom: float = CGT_SUPPORT_HEADROOM, ) -> UKCGTSupportSplitResult: """Split the wealthiest households of each income band into light copies. Deterministic and mass-conserving: no draw, no seed, no salt. Each - selected household of weight ``w`` becomes ``n = ceil(w / 60)`` copies at - ``w / n``; the root keeps its ids, copy ``k`` takes every entity id plus - ``k`` times the declared radix of the copy-id table's - ``cgt_support_split`` field (so every incoming id must sit below it, and - ``n`` is at most the field's bound of 10,000), records ``k`` in - ``cgt_support_copy_index`` and carries a full copy of the household's - person and benefit-unit rows. The exact-total correction lands on the - heaviest unselected incumbent so every copy weight stays bitwise - ``w / n``. Refuses a frame the clone stage has already seen or one this - stage has already split. + selected household of weight ``w`` becomes ``n = ceil(K x w / 60)`` + copies at ``w / n``, ``K`` the frame's geographic clone count + (``clone_count``, which the stage transform reads from the frame), so + ``n`` is the count the household's source gets on a one-clone spine and + every copy weighs at most ``60 / K`` (:func:`cgt_support_copy_counts`). + The selection walk still accumulates each row's own weight. The root + keeps its ids, copy ``k`` takes every entity id plus ``k`` times the + declared radix of the copy-id table's ``cgt_support_split`` field (so + every incoming id must sit below it, and ``n`` is at most the field's + bound of 10,000), records ``k`` in ``cgt_support_copy_index`` and carries + a full copy of the household's person and benefit-unit rows. The + exact-total correction lands on the heaviest unselected incumbent so + every copy weight stays bitwise ``w / n``. Refuses a frame the clone + stage has already seen or one this stage has already split. """ validate_uk_national_frame(frame) + clone_count = _require_clone_count(clone_count) person = frame.table("person").copy() benunit = frame.table("benunit").copy() household = frame.table("household").copy() @@ -511,7 +625,7 @@ def split_cgt_support_households( ) copies = np.ones(len(household), dtype="int64") copies[selected] = cgt_support_copy_counts( - weights[selected], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT + weights[selected], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=clone_count ) _refuse_copies_beyond_the_copy_id_field(copies) for table, columns in ( @@ -637,6 +751,7 @@ def split_cgt_support_households( new_total=float(exact.total), frs_selected=frs_selected, spi_selected=spi_selected, + clone_count=clone_count, ) @@ -713,8 +828,8 @@ def cgt_support_split_operation_parameters() -> tuple[ ), "maximum_copy_weight": CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, "copy_rule": ( - "copies = ceil(household weight / maximum_copy_weight); every " - "copy carries weight / copies" + "copies = ceil(geographic clone count x household weight / " + "maximum_copy_weight); every copy carries weight / copies" ), "id_remapping": ( "copy-id table field cgt_support_split; copy k takes every " diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index d617df3c14..4809d5c549 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -1812,6 +1812,11 @@ def _cgt_support_split_gate( Exhaustion is a recorded regime, not a failure (microcosm#1045): a column whose pool is lighter than its support mass selects the whole pool and says so. The split assigns no value, so nothing distributional is held. + The split counts copies on the receipt's geographic clone count ``K`` + times each household's weight (microcosm#931), so both weight checks + compare ``K`` times the receipted weight with the maximum: every copy + weighs at most the maximum over ``K``. At ``K = 1`` the checks and their + messages are unchanged. """ check = "cgt_support_split" @@ -1840,6 +1845,18 @@ def _cgt_support_split_gate( failures.append( f"{stage}: receipt {key} {value} differs from the gate's {expected}." ) + clone_count = declared.get("clone_count") + if ( + not isinstance(clone_count, int) + or isinstance(clone_count, bool) + or clone_count < 1 + ): + raise ValueError(f"{stage}.parameters.clone_count must be a positive integer.") + over_clones = ( + "" + if clone_count == 1 + else f" divided by the geographic clone count {clone_count}" + ) mass = _mapping(evidence.get("mass"), label=f"{stage}.mass") old_total = _finite_number(mass.get("old_total"), label=f"{stage}.mass.old_total") @@ -1902,10 +1919,10 @@ def _cgt_support_split_gate( f"{stage}: column from {lower} support mass {support} differs from " f"{clone_split_factor} x {headroom} x {published}." ) - if row_heaviest_copy > maximum_copy_weight * (1.0 + 1e-12): + if clone_count * row_heaviest_copy > maximum_copy_weight * (1.0 + 1e-12): failures.append( f"{stage}: column from {lower} copy weight {row_heaviest_copy} " - f"exceeds the maximum {maximum_copy_weight}." + f"exceeds the maximum {maximum_copy_weight}{over_clones}." ) if pool_exhausted: exhausted.append(lower) @@ -1921,10 +1938,14 @@ def _cgt_support_split_gate( f"{stage}: column from {lower} selected mass {selected} falls short " f"of its support mass {support} without recording exhaustion." ) - if heaviest_selected <= maximum_copy_weight and counts["copies_created"]: + if ( + clone_count * heaviest_selected <= maximum_copy_weight + and counts["copies_created"] + ): failures.append( f"{stage}: column from {lower} created {counts['copies_created']} " - "copies although no selected household exceeds the maximum weight." + "copies although no selected household exceeds the maximum weight" + f"{over_clones}." ) sums["support_mass"] += support sums["selected_mass"] += selected diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index d2002f4819..f3096d065f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -41,6 +41,7 @@ cgt_support_income_band, cgt_support_mass_by_income_band, cgt_support_split_operation_parameters, + geographic_clone_count, select_cgt_support_households, split_cgt_support_households, ) @@ -50,6 +51,7 @@ load_hmrc_cgt_joint_distribution, ) from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate from microcosm.frame import WeightKind from test_support.microcosm_build.uk_cgt_support import ( BAND_INCOMES, @@ -62,6 +64,7 @@ ) from test_support.microcosm_build.uk_clone_first_frames import ( assert_copy_ids_follow_lineage, + clone_first_frame, with_last_household_moved, ) @@ -120,6 +123,7 @@ def _scenario_split(**overrides): _scenario_frame(**overrides), distribution=support_distribution(SCENARIO_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) @@ -445,18 +449,228 @@ def test_selection_refuses_inconsistent_inputs(): def test_copy_counts_are_integer_ceilings_of_at_least_one(): counts = cgt_support_copy_counts( - [0.0, 1.0, 60.0, 60.0001, 120.0, 121.0, 600.0], 60.0 + [0.0, 1.0, 60.0, 60.0001, 120.0, 121.0, 600.0], 60.0, clone_count=1 ) assert counts.tolist() == [1, 1, 1, 2, 2, 3, 10] assert counts.dtype == np.dtype("int64") - assert cgt_support_copy_counts([59.0]).tolist() == [ + assert cgt_support_copy_counts([59.0], clone_count=1).tolist() == [ math.ceil(59.0 / CGT_SUPPORT_MAXIMUM_COPY_WEIGHT) ] with pytest.raises(ValueError, match="maximum copy weight"): - cgt_support_copy_counts([1.0], 0.0) + cgt_support_copy_counts([1.0], 0.0, clone_count=1) with pytest.raises(ValueError, match="finite"): - cgt_support_copy_counts([np.nan], 60.0) + cgt_support_copy_counts([np.nan], 60.0, clone_count=1) + + +# --- geographic clones (microcosm#931) ----------------------------------------- +# +# A geographic clone of the clone-first spine carries 1/K of its source +# household's weight, so the copy count reads K x the row's weight: every +# clone of a household gets the count its source gets at K = 1, and each copy +# weighs at most 60 / K. Every weight here is synthetic. + + +def _todays_copy_counts(weights) -> np.ndarray: + """The rule before the clone count, spelled out: ceil(w / 60), at least 1.""" + + values = np.asarray(weights, dtype=float) + return np.maximum( + np.ceil(values / CGT_SUPPORT_MAXIMUM_COPY_WEIGHT).astype("int64"), 1 + ) + + +def _edge_weights() -> np.ndarray: + """Exact multiples of 60, one ulp either side, tiny, large and random.""" + + multiples = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * np.arange(1.0, 20_001.0) + rng = np.random.default_rng(931) + return np.concatenate( + [ + [0.0, 5e-324, 1e-300, 1e-12, 0.5, 59.999999, 60.0, 60.000001], + multiples, + np.nextafter(multiples, np.inf), + np.nextafter(multiples, 0.0), + CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * np.asarray([10_000.0, 1e6, 1e9, 1e12]), + rng.uniform(0.0, 600_000.0, 200_000), + rng.lognormal(5.0, 2.0, 200_000), + ] + ) + + +def test_copy_counts_at_one_clone_are_todays_ceiling_bit_for_bit(): + weights = _edge_weights() + + counts = cgt_support_copy_counts(weights, clone_count=1) + + assert counts.dtype == np.dtype("int64") + np.testing.assert_array_equal(counts, _todays_copy_counts(weights)) + # The edges the snap would have moved: one ulp above a multiple of 60 is + # rounded up at one clone, as before. + above = np.nextafter(CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * 2.0, np.inf) + assert cgt_support_copy_counts([above], clone_count=1).tolist() == [3] + + +@pytest.mark.parametrize("clone_count", [3, 7, 15]) +def test_copy_counts_of_clone_replicas_are_their_sources_counts(clone_count: int): + # Exact multiples of 60, one ulp below them and random weights: each + # replica at W / K gets the count its source W gets at one clone. + multiples = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * np.arange(1.0, 20_001.0) + rng = np.random.default_rng(clone_count) + sources = np.concatenate( + [ + multiples, + np.nextafter(multiples, 0.0), + rng.uniform(0.0, 600_000.0, 200_000), + rng.lognormal(5.0, 2.0, 200_000), + ] + ) + replicas = sources / clone_count + + counts = cgt_support_copy_counts(replicas, clone_count=clone_count) + + np.testing.assert_array_equal( + counts, cgt_support_copy_counts(sources, clone_count=1) + ) + # Every copy of a replica weighs at most 60 / K, to the gate's tolerance. + assert ( + clone_count * (replicas / counts) + <= CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * (1.0 + 1e-12) + ).all() + + +def test_the_snap_keeps_exact_multiples_exact_at_seven_clones(): + # At K = 7 the replica 60m / 7 times 7 lands an ulp above 60m for some m + # (the verifier's case): the unsnapped ceiling adds a copy there. + multiples = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * np.arange(1.0, 20_001.0) + replicas = multiples / 7 + unsnapped = np.ceil((7 * replicas) / CGT_SUPPORT_MAXIMUM_COPY_WEIGHT) + rounded_up = unsnapped != np.arange(1.0, 20_001.0) + assert rounded_up.sum() > 0 + assert ( + cgt_support_copy_counts(replicas[rounded_up], clone_count=7).tolist() + == np.arange(1, 20_001)[rounded_up].tolist() + ) + + +def test_the_snap_counts_a_quotient_ulps_above_an_integer_as_that_integer(): + # The case the snap gives up, as documented: a source one ulp above 120 + # makes 3 copies at one clone, but its replicas count 2 at K > 1. + source = np.nextafter(2.0 * CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, np.inf) + assert cgt_support_copy_counts([source], clone_count=1).tolist() == [3] + for clone_count in (2, 3, 7): + replica = source / clone_count + assert cgt_support_copy_counts([replica], clone_count=clone_count).tolist() == [ + 2 + ] + + +@pytest.mark.parametrize("clone_count", [0, -1, True, 1.0, 2.5, None, "3"]) +def test_copy_counts_refuse_a_clone_count_that_is_not_a_positive_integer( + clone_count, +): + with pytest.raises(ValueError, match="positive integer geographic clone count"): + cgt_support_copy_counts([1.0], clone_count=clone_count) + with pytest.raises(ValueError, match="positive integer geographic clone count"): + _scenario_split_at(clone_count) + + +def _scenario_split_at(clone_count): + return split_cgt_support_households( + _scenario_frame(), + distribution=support_distribution(SCENARIO_TOP_CELLS), + parameters=PARAMETERS, + clone_count=clone_count, + headroom=SCENARIO_HEADROOM, + ) + + +def test_the_clone_index_column_is_the_lineages_geographic_branch(): + from microcosm.build.uk_runtime.cgt_support import _GEOGRAPHIC_CLONE_INDEX_COLUMN + from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_SUPPORT_BRANCH, + UK_COPY_LINEAGE, + ) + from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column + + (branch,) = [ + branch + for branch in UK_COPY_LINEAGE.branches + if branch.name == GEOGRAPHIC_SUPPORT_BRANCH + ] + assert ( + _GEOGRAPHIC_CLONE_INDEX_COLUMN + == ladder_clone_index_column("household") + == branch.index + == "household_clone_index" + ) + + +def test_geographic_clone_count_is_one_more_than_the_largest_index(): + assert geographic_clone_count(pd.DataFrame({"household_id": [1, 2]})) == 1 + for indices, expected in (([0, 0], 1), ([0, 1, 2, 1, 0], 3), ([2, 0, 1], 3)): + household = pd.DataFrame( + {"household_clone_index": np.asarray(indices, dtype="int64")} + ) + assert geographic_clone_count(household) == expected + empty = pd.DataFrame({"household_clone_index": np.zeros(0, dtype="int64")}) + assert geographic_clone_count(empty) == 1 + + +@pytest.mark.parametrize( + "values,message", + [ + ( + np.asarray([0, 2, 2, 0], dtype="int64"), + "1 of the 3 indices from 0 to the largest are carried by no household; " + "the CGT support split counts K geographic clones as the " + "'household_clone_index' values 0 to K - 1 (microcosm#931).", + ), + ( + np.asarray([0, 3], dtype="int64"), + "2 of the 4 indices from 0 to the largest are carried by no household; " + "the CGT support split counts K geographic clones as the " + "'household_clone_index' values 0 to K - 1 (microcosm#931).", + ), + ( + np.asarray([0, -1, 1, -2], dtype="int64"), + "2 household(s) carry a negative 'household_clone_index'; the CGT " + "support split counts K geographic clones as the " + "'household_clone_index' values 0 to K - 1 (microcosm#931).", + ), + ( + pd.array([0, None, 1], dtype="Int64"), + "1 household(s) carry no 'household_clone_index'; the CGT support " + "split counts K geographic clones as the 'household_clone_index' " + "values 0 to K - 1 (microcosm#931).", + ), + ( + np.asarray([0.0, 1.0]), + "The CGT support split counts geographic clones on an integer " + "'household_clone_index' column; it has dtype float64.", + ), + ( + np.asarray([False, True]), + "The CGT support split counts geographic clones on an integer " + "'household_clone_index' column; it has dtype bool.", + ), + ], +) +def test_geographic_clone_count_refuses_indices_other_than_0_to_k_by_count( + values, message: str +): + household = pd.DataFrame( + { + "household_id": 1_000_003 + np.arange(len(values)), + "household_clone_index": values, + } + ) + + with pytest.raises(ValueError) as caught: + geographic_clone_count(household) + + assert str(caught.value) == message + assert "1000003" not in str(caught.value) # --- the split ---------------------------------------------------------------- @@ -509,6 +723,7 @@ def test_split_offsets_every_entity_id_by_k_times_the_declared_radix(): frame, distribution=support_distribution(SCENARIO_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) out_person = result.frame.table("person") @@ -562,6 +777,7 @@ def test_split_copies_take_their_roots_ids_plus_k_times_the_declared_radix(): frame, distribution=support_distribution(SCENARIO_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) @@ -634,6 +850,7 @@ def test_split_copies_of_spi_rows_take_their_parents_ids_plus_k_times_the_radix( frame, distribution=support_distribution(SCENARIO_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) @@ -677,6 +894,7 @@ def test_a_household_of_exactly_the_fields_bound_in_rows_fits(): copies = cgt_support_copy_counts( [CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * bound, 1.0], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + clone_count=1, ) assert copies.tolist() == [bound, 1] _refuse_copies_beyond_the_copy_id_field(copies) @@ -698,6 +916,7 @@ def test_split_refuses_incoming_ids_at_its_copy_id_radix_by_count(): frame, distribution=support_distribution(SCENARIO_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) assert str(7 + radix("cgt_support_split")) not in str(caught.value) @@ -749,6 +968,7 @@ def test_split_copies_every_column_and_membership_unchanged(): frame, distribution=support_distribution(SCENARIO_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) person = result.frame.table("person") @@ -855,6 +1075,7 @@ def test_split_records_pool_exhaustion_and_empty_bands_without_refusing(): {0: {250_000: 1_000.0}, 125_140: {250_000: 10.0}} ), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) bands = {row["income_lower_bound"]: row for row in result.band_rows} @@ -894,6 +1115,7 @@ def test_split_falls_back_when_every_incumbent_is_selected(): frame, distribution=support_distribution({0: {250_000: 1_000.0}}), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) @@ -912,6 +1134,7 @@ def test_split_counts_suppressed_cells_and_reports_channels_when_present(): {0: {250_000: None, 500_000: None, 1_000_000: 5.0}} ), parameters=PARAMETERS, + clone_count=1, headroom=SCENARIO_HEADROOM, ) @@ -947,6 +1170,7 @@ def test_split_refuses_a_cloned_or_already_split_frame(): frame, distribution=support_distribution(SCENARIO_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, ) @@ -967,6 +1191,7 @@ def _split(person_table, household_table): frame, distribution=support_distribution(SCENARIO_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, ) with pytest.raises(ValueError, match="lacks"): @@ -1006,10 +1231,13 @@ def test_evidence_is_json_serialisable_with_plain_scalars(): "zero_weight_excluded": 1, } assert evidence["mass"] == {"old_total": 502.0, "new_total": 502.0} + # The geographic clone count is the one new receipt key (microcosm#931); + # a frame without the clone index is one clone. assert evidence["parameters"] == { "clone_split_factor": 2, "headroom": 2.0, "maximum_copy_weight": 60.0, + "clone_count": 1, } assert evidence["support_channel_split"] == {"frs": 2, "spi": 1} assert [row["income_lower_bound"] for row in evidence["bands"]] == list( @@ -1046,6 +1274,210 @@ def _leaves(value): ) +# --- the split on clone-first frames (microcosm#931) ---------------------------- +# +# Nine one-person households. Band 0 holds exact multiples of 60 that the +# unsnapped rule overcounts at seven clones (900, 1,800, 4,020 and 7,620: 15, +# 30, 67 and 127 copies), a weight one ulp below 300 (5 copies), a light one +# (40, uncopied) and a non-multiple (1,234.5, 21 copies); the zero-weight +# household is outside every pool. The synthetic joint asks band 0 for far +# more than its pool, so the whole pool is selected at every clone count and +# the selection nests exactly; band 37,700 asks for nothing, so its household +# takes the exact-total correction (on clone 0, the first heaviest), heavy +# enough (9,000) that the correction moves it far less than the weights' +# nesting tolerance. + +NESTING_WEIGHTS = [ + 900.0, + 1_800.0, + 4_020.0, + 7_620.0, + float(np.nextafter(300.0, 0.0)), + 40.0, + 1_234.5, + 9_000.0, + 0.0, +] +NESTING_WEALTH = [7e3, 6e3, 5e3, 4e3, 3e3, 2e3, 1e3, 1.0, 1e6] +NESTING_INCOMES = [BAND_INCOMES[0]] * 7 + [BAND_INCOMES[37_700], BAND_INCOMES[0]] +NESTING_TOP_CELLS = {0: {250_000: 1_000_000.0}} +NESTING_COPIES = [15, 30, 67, 127, 5, 1, 21, 1, 1] + + +def _clone_first_split(clone_count: int, *, weights=NESTING_WEIGHTS): + """The transform on the nesting scenario as ``clone_count`` clones.""" + + frame = clone_first_frame( + support_frame(weights=weights, wealth=NESTING_WEALTH, incomes=NESTING_INCOMES), + clone_count, + ) + transform = UKCGTSupportSplitStageTransform( + stage=support_stage(), + distribution=support_distribution(NESTING_TOP_CELLS), + parameters=PARAMETERS, + ) + return transform(frame), transform.last_result + + +def _families(frame) -> pd.DataFrame: + household = frame.table("household") + return pd.DataFrame( + { + "source": household["source_household_id"].to_numpy(), + "clone": household["household_clone_index"].to_numpy(), + "copies": household[CGT_SUPPORT_COPIES_COLUMN].to_numpy(), + "weight": np.asarray(frame.weights_for("household").values), + } + ) + + +def test_split_at_one_indexed_clone_is_the_split_without_the_index(): + plain = _scenario_split() + indexed = split_cgt_support_households( + clone_first_frame(_scenario_frame(), 1), + distribution=support_distribution(SCENARIO_TOP_CELLS), + parameters=PARAMETERS, + clone_count=1, + headroom=SCENARIO_HEADROOM, + ) + _, through_transform = _clone_first_split(1) + + assert indexed.evidence() == plain.evidence() + assert indexed.frame.weights_for("household").values.tolist() == ( + plain.frame.weights_for("household").values.tolist() + ) + assert through_transform.evidence()["parameters"]["clone_count"] == 1 + + +@pytest.mark.parametrize("clone_count", [3, 7]) +def test_split_gives_every_geographic_clone_its_sources_copies(clone_count: int): + one_frame, one = _clone_first_split(1) + many_frame, many = _clone_first_split(clone_count) + reference = _families(one_frame) + families = _families(many_frame) + + # The transform read K from the frame, and every count scales by K. + assert many.evidence()["parameters"]["clone_count"] == clone_count + assert one.copies_created == sum(n - 1 for n in NESTING_COPIES) == 259 + assert ( + many.households_selected + == clone_count * one.households_selected + == (clone_count * 7) + ) + assert many.copies_created == clone_count * one.copies_created + rows = families.groupby(["source", "clone"]).size() + expected = dict(zip(range(1, 10), NESTING_COPIES, strict=True)) + for (source, clone), count in rows.items(): + # Each clone of a household is divided as its source is at K = 1. + assert count == expected[source], (source, clone) + assert sorted(set(families["clone"])) == list(range(clone_count)) + for source, n in expected.items(): + member = families["source"] == source + assert (families.loc[member, "copies"] == n).all() + # Nested weights: K x every clone's rows is the source's rows. + source_weights = np.sort( + reference.loc[reference["source"] == source, "weight"].to_numpy() + ) + for clone in range(clone_count): + clone_weights = np.sort( + families.loc[member & (families["clone"] == clone), "weight"] + ) + np.testing.assert_allclose( + clone_count * clone_weights, source_weights, rtol=1e-12, atol=0.0 + ) + # Every copy, root included, weighs at most 60 / K. + split = families["copies"] > 1 + assert ( + clone_count * families.loc[split, "weight"] + <= CGT_SUPPORT_MAXIMUM_COPY_WEIGHT * (1.0 + 1e-12) + ).all() + assert many_frame.weights_for("household").total == pytest.approx( + one_frame.weights_for("household").total, rel=1e-12 + ) + if clone_count == 7: + # Without the snap, the four multiples of 60 would gain a copy here. + replicas = np.asarray(NESTING_WEIGHTS[:4]) / 7 + unsnapped = np.ceil((7 * replicas) / CGT_SUPPORT_MAXIMUM_COPY_WEIGHT) + assert unsnapped.tolist() == [n + 1 for n in NESTING_COPIES[:4]] + + +def test_the_transform_refuses_a_gap_in_the_clone_indices_by_count(): + frame = clone_first_frame(_scenario_frame(), 2) + household = frame.table("household").copy() + household.loc[household["household_clone_index"] == 1, "household_clone_index"] = 2 + gapped = uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=household, + household_weights=frame.weights_for("household").values, + time_period="2024", + ) + transform = UKCGTSupportSplitStageTransform( + stage=support_stage(), + distribution=support_distribution(SCENARIO_TOP_CELLS), + parameters=PARAMETERS, + ) + + with pytest.raises(ValueError) as caught: + transform(gapped) + + # A count and the rule, never a household. + assert str(caught.value) == ( + "1 of the 3 indices from 0 to the largest are carried by no household; " + "the CGT support split counts K geographic clones as the " + "'household_clone_index' values 0 to K - 1 (microcosm#931)." + ) + assert transform.last_result is None + + +def _packaged_split_gate_parameters() -> dict[str, object]: + from importlib.resources import files + + gates = json.loads( + files("microcosm.build.uk").joinpath("gates.json").read_text(encoding="utf-8") + )["gates"] + (gate,) = [g for g in gates if g["id"] == "uk_stage_cgt_support_split_mass"] + return dict(gate["parameters"]) + + +def test_the_stage_gate_reads_the_receipts_clone_count(): + # Three clones of households at 150, 120 and 100: every clone weighs at + # most 60 (50, 40 and 33.3), but three times its weight does not, so the + # copies are made. The packaged gate passes the receipt and would fail it + # if it ignored the clone count. + weights = [150.0, 120.0, 100.0, 0.0, 0.0, 0.0, 0.0, 90.0, 0.0] + frame, result = _clone_first_split(3, weights=weights) + evidence = result.evidence() + parameters = _packaged_split_gate_parameters() + assert evidence["parameters"]["clone_count"] == 3 + assert max(row["heaviest_selected_weight"] for row in evidence["bands"]) == 50.0 + assert evidence["totals"]["copies_created"] == 3 * (2 + 1 + 1) + + passed = uk_stage_health_gate( + evidence=evidence, + stage="cgt_support_split", + check="cgt_support_split", + parameters=parameters, + ) + ignored = uk_stage_health_gate( + evidence={ + **evidence, + "parameters": {**evidence["parameters"], "clone_count": 1}, + }, + stage="cgt_support_split", + check="cgt_support_split", + parameters=parameters, + ) + + assert passed.passed, passed.failures + assert ignored.passed is False + assert ignored.failures == ( + "cgt_support_split: column from 0.0 created 12 copies although no selected " + "household exceeds the maximum weight.", + ) + assert frame.weights_for("household").total == pytest.approx(460.0, rel=1e-12) + + # --- the closed-world dictionary and the transform ------------------------------ @@ -1083,6 +1515,10 @@ def test_operation_dictionary_restates_the_reviewed_design(): UK_CGT_INVESTABLE_WEALTH_COLUMNS ) assert payload["maximum_copy_weight"] == 60.0 + assert payload["copy_rule"] == ( + "copies = ceil(geographic clone count x household weight / " + "maximum_copy_weight); every copy carries weight / copies" + ) assert payload["flag_column"] == HOUSEHOLD_IS_CGT_SUPPORT_COPY assert payload["copies_column"] == CGT_SUPPORT_COPIES_COLUMN assert payload["copy_index_column"] == CGT_SUPPORT_COPY_INDEX_COLUMN diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index e3d9d97ae6..681c601de1 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -33,6 +33,7 @@ from microcosm.build.uk_runtime.cgt_support import ( CGT_SUPPORT_COPIES_COLUMN, HOUSEHOLD_IS_CGT_SUPPORT_COPY, + geographic_clone_count, split_cgt_support_households, ) from microcosm.build.uk_runtime.etb_services import ( @@ -48,6 +49,7 @@ support_distribution, support_frame, ) +from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -755,6 +757,7 @@ def _split_then_cloned(): ), distribution=support_distribution(_SPLIT_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, ) cloned = clone_cgt_incidence( split.frame, distribution=load_advani_summers_distribution() @@ -1056,6 +1059,124 @@ def test_an_artifact_without_the_split_layer_is_refused(self) -> None: self._receipt(frame) +#: Both income bands of the split scenario ask for far more than their pools, +#: so every positive-weight household is selected at every clone count. +_EXHAUSTING_TOP_CELLS = {0: {250_000: 1e6}, 37_700: {250_000: 1e6}} + + +def _clone_first_split_then_cloned(clone_count: int): + """The split scenario as ``clone_count`` geographic clones, split then cloned.""" + + frame = clone_first_frame( + support_frame( + weights=_SPLIT_WEIGHTS, wealth=_SPLIT_WEALTH, incomes=_SPLIT_INCOMES + ), + clone_count, + ) + split = split_cgt_support_households( + frame, + distribution=support_distribution(_EXHAUSTING_TOP_CELLS), + parameters=PARAMETERS, + clone_count=geographic_clone_count(frame.table("household")), + ) + cloned = clone_cgt_incidence( + split.frame, distribution=load_advani_summers_distribution() + ) + return split, cloned.frame + + +def _exhausting_receipt(frame) -> tuple[dict[str, object], dict[str, object]]: + problems: dict[str, object] = {} + receipt = _load_tool()._e8_support_split( + frame, + problems, + distribution=support_distribution(_EXHAUSTING_TOP_CELLS), + parameters=PARAMETERS, + permutation_seed=7, + ) + return receipt, problems + + +class TestSupportSplitRecomputeAcrossGeographicClones: + """The recompute counts copies on the frame's geographic clone count. + + The stage reads ``K`` from its input and counts ``ceil(K x w / 60)`` + copies (microcosm#931); the tool reads ``K`` from the pre-split table the + same way. At one clone its receipt is the one the rule before ``K`` gave. + """ + + def test_at_one_clone_the_receipt_is_the_rule_before_the_clone_count( + self, monkeypatch + ) -> None: + from microcosm.build.uk_runtime import cgt_support + + _, frame = _split_then_cloned() + receipt, problems = TestE8SupportSplitRecompute._receipt(frame) + + def ceiling_before_the_clone_count( + weights, maximum_copy_weight, *, clone_count + ): + assert clone_count == 1 + values = np.asarray(weights, dtype=float) + return np.maximum( + np.ceil(values / float(maximum_copy_weight)).astype("int64"), 1 + ) + + monkeypatch.setattr( + cgt_support, "cgt_support_copy_counts", ceiling_before_the_clone_count + ) + before, before_problems = TestE8SupportSplitRecompute._receipt(frame) + + assert problems == before_problems == {} + assert receipt == before + # No key is new: the clone count is a stage receipt key only. + assert set(receipt) == { + "mass", + "id_multiplier", + "pre_split_households", + "copies", + "copy_index_column_stored", + "families", + "max_abs_family_weight_diff", + "bands", + "support_mass", + "households_selected", + "copies_recomputed", + } + + def test_at_three_clones_the_recompute_matches_the_stored_layer(self) -> None: + one_split, one = _clone_first_split_then_cloned(1) + three_split, three = _clone_first_split_then_cloned(3) + assert one_split.evidence()["parameters"]["clone_count"] == 1 + assert three_split.evidence()["parameters"]["clone_count"] == 3 + + one_receipt, one_problems = _exhausting_receipt(one) + three_receipt, three_problems = _exhausting_receipt(three) + + assert one_problems == three_problems == {} + # Five of the six households are divided at one clone (30 is not); + # each of their three clones is divided the same way. + assert one_receipt["families"] == 5 + for key in ("families", "copies", "copies_recomputed", "pre_split_households"): + assert three_receipt[key] == 3 * one_receipt[key], key + + def test_a_recompute_that_ignored_the_clone_count_would_report_it( + self, monkeypatch + ) -> None: + from microcosm.build.uk_runtime import cgt_support + + _, three = _clone_first_split_then_cloned(3) + monkeypatch.setattr(cgt_support, "geographic_clone_count", lambda household: 1) + + _, problems = _exhausting_receipt(three) + + # At one third of their weight no clone needs a copy, so none of the + # 15 stored families is recomputed. + assert problems == { + "support_split_selection_stored": {"missing": 15, "extra": 0} + } + + # --- copy ids at the copy-id table's declared radices (microcosm#931) ---------- # # The decoders read every copy id at its field's declared radix, never at a @@ -1316,6 +1437,7 @@ def _residentially_split(): ), distribution=support_distribution(_SPLIT_TOP_CELLS), parameters=PARAMETERS, + clone_count=1, ) twinned = clone_cgt_incidence( split.frame, distribution=load_advani_summers_distribution() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py index 99e4b8f96a..eebc06790e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py @@ -410,6 +410,7 @@ def _support_split_receipt() -> dict[str, object]: "clone_split_factor": 2, "headroom": 2.0, "maximum_copy_weight": 60.0, + "clone_count": 1, }, } @@ -518,6 +519,88 @@ def test_cgt_support_split_gate_refuses_a_missing_field() -> None: _support_split_gate(evidence) +def test_cgt_support_split_gate_messages_are_unchanged_at_one_clone() -> None: + """At one geographic clone both weight checks read as before microcosm#931.""" + + copy_over = _support_split_receipt() + copy_over["bands"][0]["heaviest_copy_weight"] = 60.5 + light = _support_split_receipt() + light["bands"][0]["heaviest_selected_weight"] = 59.0 + + assert _support_split_gate(copy_over).failures == ( + "cgt_support_split: column from 0.0 copy weight 60.5 exceeds the maximum 60.0.", + ) + assert _support_split_gate(light).failures == ( + "cgt_support_split: column from 0.0 created 1120 copies although no " + "selected household exceeds the maximum weight.", + ) + + +def _three_clone_receipt() -> dict[str, object]: + """The receipt at three geographic clones (microcosm#931). + + The split counts copies on three times each row's weight: every selected + row here weighs at most 60 while three times it does not, so copies were + made, and every copy weighs at most 20. + """ + + evidence = _support_split_receipt() + evidence["parameters"]["clone_count"] = 3 + for row in evidence["bands"]: + row["heaviest_selected_weight"] = 30.0 + row["heaviest_copy_weight"] = 18.0 + return evidence + + +def test_cgt_support_split_gate_scales_both_weight_checks_by_the_clone_count() -> None: + assert _passed(_support_split_gate(_three_clone_receipt())) + + # Read at one clone, the same rows would need no copy: ignoring the count + # fails every column falsely. + ignored = _three_clone_receipt() + ignored["parameters"]["clone_count"] = 1 + result = _support_split_gate(ignored) + assert result.passed is False + assert len(result.failures) == 4 + assert all( + failure.endswith( + "created 1120 copies although no selected household exceeds the " + "maximum weight." + ) + for failure in result.failures + ) + + # A copy of 25 is within 60 but not within 60 / 3. + heavy = _three_clone_receipt() + heavy["bands"][0]["heaviest_copy_weight"] = 25.0 + assert _support_split_gate(heavy).failures == ( + "cgt_support_split: column from 0.0 copy weight 25.0 exceeds the maximum " + "60.0 divided by the geographic clone count 3.", + ) + # Three times a selected row of 20 is the maximum, which needs no copy. + light = _three_clone_receipt() + light["bands"][1]["heaviest_selected_weight"] = 20.0 + assert _support_split_gate(light).failures == ( + "cgt_support_split: column from 37700.0 created 1120 copies although no " + "selected household exceeds the maximum weight divided by the geographic " + "clone count 3.", + ) + + +@pytest.mark.parametrize("clone_count", ["absent", 0, -3, True, 3.0, "3", None]) +def test_cgt_support_split_gate_refuses_a_receipt_without_a_clone_count( + clone_count, +) -> None: + evidence = _support_split_receipt() + if clone_count == "absent": + del evidence["parameters"]["clone_count"] + else: + evidence["parameters"]["clone_count"] = clone_count + + with pytest.raises(ValueError, match=r"parameters\.clone_count must be a positive"): + _support_split_gate(evidence) + + def test_age_tail_relative_deviation_parameter_is_live() -> None: evidence = { "stage": "uk_age_tail_disaggregation", diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 4fa04edfb2..ead642a0d9 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -563,7 +563,7 @@ ], "grain": "household", "nonnegative_outputs": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 before the clone (30 after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", "operations": [ { "adult_minimum_age": 16, @@ -573,7 +573,7 @@ "conservation": "exact_total", "copies_column": "cgt_support_copies", "copy_index_column": "cgt_support_copy_index", - "copy_rule": "copies = ceil(household weight / maximum_copy_weight); every copy carries weight / copies", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies", "declared_factor": 1, "expected_support_mass": 145000, "flag_column": "household_is_cgt_support_copy", diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 191c66627e..20fa57da78 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"48b7648953aabc721c6672038770b18a1ff7c308d71bd982770b8a3fe259280f","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"85696cd009ce349a132f753c55cb2168ed2f548de9f628ce74a71e99a0e9e6ba","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"5dcae0fb0c93d443b727ea728c5e61e11cdc4a1ad6f01f728b58eb8419760629","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"48b7648953aabc721c6672038770b18a1ff7c308d71bd982770b8a3fe259280f","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"85696cd009ce349a132f753c55cb2168ed2f548de9f628ce74a71e99a0e9e6ba","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"29386196b3832b90f7037593ab46b47af254b6153b544225f6e38f492e7103bc","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} diff --git a/test_support/microcosm_build/uk_clone_first_frames.py b/test_support/microcosm_build/uk_clone_first_frames.py index 4774d981cf..fb6fe821a6 100644 --- a/test_support/microcosm_build/uk_clone_first_frames.py +++ b/test_support/microcosm_build/uk_clone_first_frames.py @@ -12,6 +12,8 @@ frame. ``rule="table"`` mints every copy through the copy-id table, as the stages do; ``rule="retired"`` replays the rule the table replaced (ten to the digit count of the largest id the layer sees), for comparison. +:func:`clone_first_frame` clones a whole synthetic frame the same way, each +clone at an equal share of its household's weight. :func:`assert_copy_ids_follow_lineage` checks a stage's output on a synthetic frame: every new household is matched to its root in the stage's input by @@ -37,6 +39,7 @@ "IdTables", "assert_copy_ids_follow_lineage", "base_id_tables", + "clone_first_frame", "clone_first_tables", "copy_chain", "copy_households", @@ -132,6 +135,63 @@ def clone_first_tables(tables: IdTables, n_clones: int) -> IdTables: ) +def clone_first_frame(frame: Frame, n_clones: int) -> Frame: + """``frame`` as ``n_clones`` geographic clones of every household, clone-major. + + Clone ``k`` repeats every person, benefit unit and household row with + each id plus ``k x 10**9`` (the copy-id table's ``geographic_support`` + field), ``household_clone_index`` ``k`` and ``source_household_id`` the + clone-0 id (added when the frame has none), at the weight ``w / + n_clones`` of its household (the same float in every clone), as the + clone-first spine will (#931 (b)). Every other column is copied + unchanged; the clones carry the frame's weight kind and time period, and + no mass record. + """ + + from microcosm.build.uk_runtime.national_frame import ( + uk_national_frame, + uk_time_period, + ) + + person = frame.table("person") + benunit = frame.table("benunit") + household = frame.table("household").copy() + if "source_household_id" not in household.columns: + household["source_household_id"] = household["household_id"].astype("int64") + household["household_clone_index"] = np.zeros(len(household), dtype=np.int64) + persons, benunits, households = [person], [benunit], [household] + for k in range(1, n_clones): + clone_person = person.copy() + for column in _PERSON_ID_COLUMNS: + clone_person[column] = copy_id_table.mint( + clone_person[column], "geographic_support", k + ) + clone_benunit = benunit.copy() + clone_benunit["benunit_id"] = copy_id_table.mint( + clone_benunit["benunit_id"], "geographic_support", k + ) + clone_household = household.copy() + clone_household["household_id"] = copy_id_table.mint( + clone_household["household_id"], "geographic_support", k + ) + clone_household["household_clone_index"] = np.full( + len(household), k, dtype=np.int64 + ) + persons.append(clone_person) + benunits.append(clone_benunit) + households.append(clone_household) + weights = frame.weights_for("household") + clone_weights = np.asarray(weights.values, dtype=float) / n_clones + return uk_national_frame( + person=pd.concat(persons, ignore_index=True), + benunit=pd.concat(benunits, ignore_index=True), + household=pd.concat(households, ignore_index=True), + household_weights=np.tile(clone_weights, n_clones), + weight_kind=weights.kind, + time_period=uk_time_period(frame), + ) + + def id_frame(tables: IdTables, *, time_period: str = "2024") -> Frame: """``tables`` as a UK national frame at unit household weights.""" diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 136437de7b..7296798265 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -842,7 +842,9 @@ def _e8_support_split( ``parameters``, and the stored wealth columns. The stored layer must agree: the households the rule divides are exactly the roots of the flagged copies (the pre-clone rows carrying ``cgt_support_copies`` above - one), each family's count is ``ceil(w / maximum_copy_weight)``, every + one), each family's count is ``ceil(K x w / maximum_copy_weight)``, ``K`` + the geographic clone count the stage read from the frame (microcosm#931; + the recompute reads it from the pre-split table the same way), every member of a family carries ``w / n`` at stage time (its clone pair sum, since the clone runs afterwards), the copies of each root are indexed ``1..n-1`` without gaps and carry the root's count, no count is below @@ -866,6 +868,7 @@ def _e8_support_split( cgt_support_household_wealth, cgt_support_income_band, cgt_support_mass_by_income_band, + geographic_clone_count, select_cgt_support_households, ) @@ -996,6 +999,9 @@ def _e8_support_split( pre_person = person.loc[ person["person_household_id"].isin(set(pre_ids.tolist())) ].reset_index(drop=True) + # The geographic clone count the copies were counted on, read from the + # pre-split table as the stage read it from its input (microcosm#931). + clone_count = geographic_clone_count(pre_household) def recompute(person_t, household_t, weights_t): ids_t = ( @@ -1012,7 +1018,9 @@ def recompute(person_t, household_t, weights_t): ) copies = np.ones(len(ids_t), dtype="int64") copies[selected] = cgt_support_copy_counts( - weights_t[selected], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT + weights_t[selected], + CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + clone_count=clone_count, ) return ( pd.DataFrame({"selected": selected, "copies": copies}, index=ids_t), @@ -1578,7 +1586,8 @@ def e8_identity_receipt( frame with the stage's own helpers - the wealthiest households of each Table 3 income band (bands from the stored income components under the policy parameters, wealth from the stored columns, support masses from - the vendored joint) divided into ``ceil(w / 60)`` copies at ``w / n`` - + the vendored joint) divided into ``ceil(K x w / 60)`` copies at ``w / n``, + ``K`` the geographic clone count - in original and permuted row order, against the stored copies, counts, family weights, flags and mass record; (3) the student-loan plan column recomputed in full (identity-keyed top-ups at the release calibration @@ -2505,8 +2514,9 @@ def _stage_time_weight_divisor(*, after_stages: Sequence[str]) -> float: still has to be added here. The capital-gains clone is not uniform once the #970 anchor has run, so its mass is restored by pair sum instead, and the capital-gains support split (microcosm#1045) is not a divisor at - all - it divides each selected household's weight among ``ceil(w / 60)`` - copies at conserved mass - so its mass is restored by folding each copy + all - it divides each selected household's weight among + ``ceil(K x w / 60)`` copies (``K`` the geographic clone count) at + conserved mass - so its mass is restored by folding each copy onto its root after the clone fold (``_pre_split_household_weights``). ``after_stages`` names only the stages that actually ran in the artifact From bf9173a74166dc50cce374aba4c46930f42ec695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:12:29 +0000 Subject: [PATCH 018/108] Follow up the review of the source-scaled CGT copy count (#931) Three reviews of 914cae416 found no must. This commit takes their should findings and the cheap nits, and corrects three claims 914cae416 made. - Identity tool at K > 1: the E8 recompute counted each family exactly, on a weight rebuilt by re-summing the family's copies. At K > 1 that sum drifts by more ulps than the count's K-ulp snap absorbs. On the commit's own nesting scenario, the tool flagged the stage's correct count of the 7,620 family in every clone at K = 11 and K = 13. At K > 1 the stored count must now lie between the counts at the folded weight times 1 - 1e-9 and 1 + 1e-9, the tolerance the tool already gives family weights. At K = 1 the check stays exact, so K = 1 receipts and problems are unchanged. The tolerance cannot tell an exact multiple's n from n + 1. The count's unit tests pin the snap, and the flags and family weights still report a moved count. copies_recomputed stays the exact recount. - Gate tolerance: test_split_gives_every_geographic_clone_its_sources_copies now runs the packaged stage gate on the transform's receipt. At K = 7 it also asserts that 7 x the heaviest copy lands above 60, so the copy check's 1e-12 tolerance stays exercised. - Texts, correcting 914cae416: - Its changelog fragment and message said every clone of a household gets its source's count. That holds for a selected clone at exactly 1/K of its source's weight. The walk still accumulates each row's own weight, so the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. The fragment and the split's docstrings now say so, and a new test pins the boundary case. - Its Tests line omitted that the clone-first split scenario exhausts every pool, so its selection is the same at every K. - The spine gates re-key because they hash the country spec as well as stage_health; the fragment now names both. The fragment also says the snap adds a copy at K = 7 only for some exact multiples. The module docstring, the clone-split factor's comment and the split's refusal sentence name the incidence clone stage (cgt_incidence_clone). geographic_clone_count's docstring says a refusal gives a count or the column's dtype. Not changed here: the snap rule itself and a mention of it in copy_rule, which wait for a ruling on the snap; the stage notes' "the clone stage", which changes when the stage's declarations are next regenerated; and the tool's cross-check of K against the stage receipt, which lands in (b). What moves: no value, at any K. cgt_support.py changes only in docstrings and a comment, so its implementation hash moves again, and the stages that bind it re-key as in 914cae416. sources.yaml, source_stages.json, the coverage manifest and the H2 fixture are untouched; the fixture holds stage contract hashes, not code hashes. The tool's verdicts move only at K > 1, which no spine runs yet. Tests: the identity tool's nesting receipts are green at K = 7, 11, 13 and 15; a count moved off a non-multiple is reported at K = 1, 3 and 11; at K = 1 a moved count on an exact multiple is still reported by the count check. Six synthetic mutations each fail a new test: the gate without its tolerance, the tool exact at every K, the tolerance at K = 1, the tolerance ignoring K, a tolerance of 1e-1 and a strict walk. The touched and related engine-free files pass (243 and 275 tests). The coverage manifest's check and the source_stages projection check pass, and ruff is clean. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 6 +- .../microcosm/build/uk_runtime/cgt_support.py | 47 ++++---- .../engine_free/uk/test_uk_cgt_support.py | 57 +++++++++ .../uk/test_uk_identity_stability_receipts.py | 113 ++++++++++++++++-- tools/verify_uk_identity_stability.py | 44 +++++-- 5 files changed, 223 insertions(+), 44 deletions(-) diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index ea7ccebb28..b7cea3f572 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,5 +1,5 @@ -The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so every clone of a household gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. +The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. -- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt is unchanged. +- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt is unchanged. Its count check stays exact at K = 1. At K > 1 it accepts a family's stored count within the family weights' relative tolerance of 1e-9, because folding the family's copies back re-sums them, which drifts by more ulps than the snap absorbs. - **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies". The stage notes give the cap as 60 divided by the geographic clone count before the incidence clone. -- **What moves.** No value moves at K = 1. On synthetic frames, the stage's output and the identity tool's receipt are unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The stage evidence gains `clone_count`. `cgt_support_split`'s stage contract moves with its texts. Its implementation hash moves, and so does `high_value_home_support`'s, which binds `cgt_support`. The spine gates hash `stage_health`, so their keys move, and with them every spine stage after the assembled gate. The coverage manifest's source digests and the H2 fixture's declarations and contract hash are regenerated. +- **What moves.** No value moves at K = 1. On synthetic frames, the stage's output and the identity tool's receipt are unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The stage evidence gains `clone_count`. `cgt_support_split`'s stage contract moves with its texts. Its implementation hash moves, and so does `high_value_home_support`'s, which binds `cgt_support`. The spine gates hash `stage_health` and the country spec, both of which moved, so their keys move, and with them every spine stage after the assembled gate. The coverage manifest's source digests and the H2 fixture's declarations and contract hash are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index a298a0a96f..928716e6e4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -16,10 +16,9 @@ ``cgt_support_copy_index`` (0 on roots and unsplit households), the explicit lineage the geography identity kernel keys on. No value moves, no draw is made, and every household's mass and the total household mass are conserved -exactly; the clone stage then -gives each copy's clone its own identity-keyed prior, and the redraw's -wealth-ranked walk places the light rows where it was already placing heavy -ones. +exactly; the incidence clone stage (``cgt_incidence_clone``) then gives each +copy's clone its own identity-keyed prior, and the redraw's wealth-ranked +walk places the light rows where it was already placing heavy ones. The support masses are read from the vendored ``hmrc_cgt_conditioning_facts.json`` (HMRC Table 3, individuals by size of gain and taxable income) through @@ -81,7 +80,8 @@ #: roots and unsplit households: the explicit lineage the geography identity #: kernel keys on (microcosm#932), so no consumer recovers ``k`` from ids. CGT_SUPPORT_COPY_INDEX_COLUMN = "cgt_support_copy_index" -#: The clone stage halves every household and only the clone twin gains. +#: The incidence clone stage (``cgt_incidence_clone``) halves every household +#: and only the clone twin gains. CGT_SUPPORT_CLONE_SPLIT_FACTOR = 2 #: Covers clones whose prior draws at or below zero and the quarter of the #: redraw's rank key the prior carries. @@ -458,7 +458,7 @@ def geographic_clone_count(household: pd.DataFrame) -> int: is one clone. Refuses a non-integer index column, a household without an index, a negative index and indices that are not exactly 0 to ``K - 1`` (an index below the largest that no household carries). Each refusal - gives a count, never an id. + gives a count or the column's dtype, never an id or a value. """ column = _GEOGRAPHIC_CLONE_INDEX_COLUMN @@ -506,9 +506,9 @@ def cgt_support_copy_counts( ``clone_count`` is the frame's geographic clone count ``K`` (:func:`geographic_clone_count`; 1 on a national spine). A geographic clone carries ``1 / K`` of its source household's weight, so ``K x w`` is - its weight scaled back to the source: every clone of a household gets the - count its source gets at ``K = 1``, and each of its copies weighs at most - ``maximum_copy_weight / K`` (microcosm#931). + its weight scaled back to the source: a clone at exactly ``1 / K`` of its + source's weight gets the count its source gets at ``K = 1``, and each of + its copies weighs at most ``maximum_copy_weight / K`` (microcosm#931). For ``K > 1`` the quotient is snapped to the nearest integer when it lies within ``K`` ulps of it. A clone's weight ``W / K`` is rounded, so when @@ -570,18 +570,23 @@ def split_cgt_support_households( Deterministic and mass-conserving: no draw, no seed, no salt. Each selected household of weight ``w`` becomes ``n = ceil(K x w / 60)`` copies at ``w / n``, ``K`` the frame's geographic clone count - (``clone_count``, which the stage transform reads from the frame), so - ``n`` is the count the household's source gets on a one-clone spine and - every copy weighs at most ``60 / K`` (:func:`cgt_support_copy_counts`). - The selection walk still accumulates each row's own weight. The root - keeps its ids, copy ``k`` takes every entity id plus ``k`` times the - declared radix of the copy-id table's ``cgt_support_split`` field (so - every incoming id must sit below it, and ``n`` is at most the field's - bound of 10,000), records ``k`` in ``cgt_support_copy_index`` and carries - a full copy of the household's person and benefit-unit rows. The - exact-total correction lands on the heaviest unselected incumbent so - every copy weight stays bitwise ``w / n``. Refuses a frame the clone - stage has already seen or one this stage has already split. + (``clone_count``, which the stage transform reads from the frame), so a + household at exactly ``1 / K`` of its source's weight gets the ``n`` its + source gets on a one-clone spine, and every copy weighs at most + ``60 / K`` (:func:`cgt_support_copy_counts`). The selection walk still + accumulates each row's own weight, so selection nests only + approximately: the household at which a band's walk reaches its support + mass, and households tied on wealth, can be selected in some clones and + not others, lowest household id first. The root keeps its ids, copy + ``k`` takes every entity id plus ``k`` times the declared radix of the + copy-id table's ``cgt_support_split`` field (so every incoming id must + sit below it, and ``n`` is at most the field's bound of 10,000), records + ``k`` in ``cgt_support_copy_index`` and carries a full copy of the + household's person and benefit-unit rows. The exact-total correction + lands on the heaviest unselected incumbent so every copy weight stays + bitwise ``w / n``. Refuses a frame the incidence clone stage + (``cgt_incidence_clone``) has already seen or one this stage has already + split. """ validate_uk_national_frame(frame) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index f3096d065f..74487511c9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -1394,11 +1394,68 @@ def test_split_gives_every_geographic_clone_its_sources_copies(clone_count: int) assert many_frame.weights_for("household").total == pytest.approx( one_frame.weights_for("household").total, rel=1e-12 ) + # The packaged stage gate passes the receipt the transform wrote. + evidence = many.evidence() + gate = uk_stage_health_gate( + evidence=evidence, + stage="cgt_support_split", + check="cgt_support_split", + parameters=_packaged_split_gate_parameters(), + ) + assert gate.passed, gate.failures if clone_count == 7: # Without the snap, the four multiples of 60 would gain a copy here. replicas = np.asarray(NESTING_WEIGHTS[:4]) / 7 unsnapped = np.ceil((7 * replicas) / CGT_SUPPORT_MAXIMUM_COPY_WEIGHT) assert unsnapped.tolist() == [n + 1 for n in NESTING_COPIES[:4]] + # Seven times a rounded copy weight of 60 / 7 lands above 60, so the + # gate passes only on its copy check's relative tolerance of 1e-12. + assert ( + 7 * max(row["heaviest_copy_weight"] for row in evidence["bands"]) + > CGT_SUPPORT_MAXIMUM_COPY_WEIGHT + ) + + +def test_the_walk_can_stop_partway_through_a_households_clones(): + # Band 0 holds households of 900, 1,800 and 600, wealthiest first, with a + # support mass of 1,500 (2 x 1.25 x 600 published). At one clone the walk + # takes the first two, at 15 and 30 copies. At three it accumulates each + # clone's own weight: the first household's three clones (900 together) + # and then the second's clone 0 (600) reach 1,500, so the second's clones + # 1 and 2 stay whole. Selection nests only approximately (microcosm#931); + # every clone the walk selects gets its source's count. + def counts_by_clone(clone_count: int): + frame = clone_first_frame( + support_frame( + weights=[900.0, 1_800.0, 600.0], + wealth=[3e3, 2e3, 1e3], + incomes=[BAND_INCOMES[0]] * 3, + ), + clone_count, + ) + transform = UKCGTSupportSplitStageTransform( + stage=support_stage(), + distribution=support_distribution({0: {250_000: 600.0}}), + parameters=PARAMETERS, + ) + families = _families(transform(frame)) + counts = families.groupby(["source", "clone"])["copies"].first() + selected = transform.last_result.households_selected + return {key: int(count) for key, count in counts.items()}, selected + + one, one_selected = counts_by_clone(1) + three, three_selected = counts_by_clone(3) + + assert one == {(1, 0): 15, (2, 0): 30, (3, 0): 1} + assert one_selected == 2 + assert three == { + **{(1, clone): 15 for clone in range(3)}, + (2, 0): 30, + (2, 1): 1, + (2, 2): 1, + **{(3, clone): 1 for clone in range(3)}, + } + assert three_selected == 4 def test_the_transform_refuses_a_gap_in_the_clone_indices_by_count(): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 681c601de1..0efb08503c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1062,20 +1062,51 @@ def test_an_artifact_without_the_split_layer_is_refused(self) -> None: #: Both income bands of the split scenario ask for far more than their pools, #: so every positive-weight household is selected at every clone count. _EXHAUSTING_TOP_CELLS = {0: {250_000: 1e6}, 37_700: {250_000: 1e6}} - - -def _clone_first_split_then_cloned(clone_count: int): - """The split scenario as ``clone_count`` geographic clones, split then cloned.""" +_SPLIT_SCENARIO = { + "weights": _SPLIT_WEIGHTS, + "wealth": _SPLIT_WEALTH, + "incomes": _SPLIT_INCOMES, + "top_cells": _EXHAUSTING_TOP_CELLS, +} +#: The nesting scenario of the CGT support tests (test_uk_cgt_support.py), +#: nine one-person households. Band 0 holds exact multiples of 60 (900, +#: 1,800, 4,020 and 7,620: 15, 30, 67 and 127 copies), a weight one ulp below +#: 300 (5 copies), a light one (40, uncopied) and a non-multiple (1,234.5, id +#: 7, 21 copies); band 0's pool is exhausted at every clone count, band +#: 37,700 asks for nothing, so its household (9,000) takes the exact-total +#: correction, and the zero-weight household is outside every pool. +_NESTING_SCENARIO = { + "weights": [ + 900.0, + 1_800.0, + 4_020.0, + 7_620.0, + float(np.nextafter(300.0, 0.0)), + 40.0, + 1_234.5, + 9_000.0, + 0.0, + ], + "wealth": [7e3, 6e3, 5e3, 4e3, 3e3, 2e3, 1e3, 1.0, 1e6], + "incomes": [BAND_INCOMES[0]] * 7 + [BAND_INCOMES[37_700], BAND_INCOMES[0]], + "top_cells": {0: {250_000: 1_000_000.0}}, +} + + +def _clone_first_split_then_cloned(clone_count: int, scenario=_SPLIT_SCENARIO): + """A scenario as ``clone_count`` geographic clones, split then cloned.""" frame = clone_first_frame( support_frame( - weights=_SPLIT_WEIGHTS, wealth=_SPLIT_WEALTH, incomes=_SPLIT_INCOMES + weights=scenario["weights"], + wealth=scenario["wealth"], + incomes=scenario["incomes"], ), clone_count, ) split = split_cgt_support_households( frame, - distribution=support_distribution(_EXHAUSTING_TOP_CELLS), + distribution=support_distribution(scenario["top_cells"]), parameters=PARAMETERS, clone_count=geographic_clone_count(frame.table("household")), ) @@ -1085,12 +1116,14 @@ def _clone_first_split_then_cloned(clone_count: int): return split, cloned.frame -def _exhausting_receipt(frame) -> tuple[dict[str, object], dict[str, object]]: +def _exhausting_receipt( + frame, scenario=_SPLIT_SCENARIO +) -> tuple[dict[str, object], dict[str, object]]: problems: dict[str, object] = {} receipt = _load_tool()._e8_support_split( frame, problems, - distribution=support_distribution(_EXHAUSTING_TOP_CELLS), + distribution=support_distribution(scenario["top_cells"]), parameters=PARAMETERS, permutation_seed=7, ) @@ -1103,6 +1136,8 @@ class TestSupportSplitRecomputeAcrossGeographicClones: The stage reads ``K`` from its input and counts ``ceil(K x w / 60)`` copies (microcosm#931); the tool reads ``K`` from the pre-split table the same way. At one clone its receipt is the one the rule before ``K`` gave. + At ``K > 1`` its count check holds the stored count to the family + weights' relative tolerance of 1e-9, since the fold re-sums the copies. """ def test_at_one_clone_the_receipt_is_the_rule_before_the_clone_count( @@ -1176,6 +1211,68 @@ def test_a_recompute_that_ignored_the_clone_count_would_report_it( "support_split_selection_stored": {"missing": 15, "extra": 0} } + @pytest.mark.parametrize("clone_count", [7, 11, 13, 15]) + def test_exact_multiples_of_60_receipt_green_at_any_clone_count( + self, clone_count: int + ) -> None: + # At 11 and 13 clones the fold re-sums the 7,620 family's 127 copies + # to a weight whose K x w / 60 sits more ulps above 127 than the + # count's snap absorbs, so an exact recount would count 128 in every + # clone; the tolerance accepts the stage's 127. + one_split, one = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) + split, many = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) + + one_receipt, one_problems = _exhausting_receipt(one, _NESTING_SCENARIO) + receipt, problems = _exhausting_receipt(many, _NESTING_SCENARIO) + + assert one_problems == problems == {} + assert one_receipt["families"] == 6 + assert one_receipt["copies"] == one_split.copies_created == 259 + assert receipt["families"] == clone_count * 6 + assert receipt["copies"] == split.copies_created == clone_count * 259 + + @pytest.mark.parametrize("clone_count", [1, 3, 11]) + def test_a_count_moved_off_a_non_multiple_is_reported( + self, clone_count: int + ) -> None: + # Household 7 (1,234.5: 21 copies, K x w / 60 = 20.575) lies far + # from an integer, so a count of 20 or 22 is outside the tolerance. + _, frame = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) + for moved in (20, 22): + household = frame.table("household").copy() + household.loc[household["household_id"] == 7, CGT_SUPPORT_COPIES_COLUMN] = ( + moved + ) + + _, problems = _exhausting_receipt( + _rebuild(frame, household=household), _NESTING_SCENARIO + ) + + assert problems["support_split_copies_stored"] == 1, moved + + def test_at_one_clone_the_count_check_stays_exact(self) -> None: + # Household 1 (900: 15 copies) moved to 16. At one clone the count + # check reports it, as before the clone count. At three, 3 x w / 60 + # is exactly 15 and the tolerance cannot tell 15 from 16; the flags + # and the family weights report the family instead. + for clone_count, counted in ((1, True), (3, False)): + _, frame = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) + household = frame.table("household").copy() + household.loc[household["household_id"] == 1, CGT_SUPPORT_COPIES_COLUMN] = ( + 16 + ) + + _, problems = _exhausting_receipt( + _rebuild(frame, household=household), _NESTING_SCENARIO + ) + + assert ("support_split_copies_stored" in problems) is counted + assert problems["support_split_flags"] == { + "copies_disagreeing_with_root": 14, + "families_with_missing_or_extra_copies": 1, + } + assert problems["support_split_family_weights"] == 15 + # --- copy ids at the copy-id table's declared radices (microcosm#931) ---------- # diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 7296798265..ffa063b0dd 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -844,15 +844,18 @@ def _e8_support_split( flagged copies (the pre-clone rows carrying ``cgt_support_copies`` above one), each family's count is ``ceil(K x w / maximum_copy_weight)``, ``K`` the geographic clone count the stage read from the frame (microcosm#931; - the recompute reads it from the pre-split table the same way), every - member of a family carries ``w / n`` at stage time (its clone pair sum, - since the clone runs afterwards), the copies of each root are indexed - ``1..n-1`` without gaps and carry the root's count, no count is below - one, and the stage's mass record conserves the total. The recompute is - repeated on permuted person and household tables and must reproduce the - divided set and the counts exactly. A selected household light enough - for a single copy leaves no trace in the artifact, so only families are - compared. Problems land under ``support_split_*`` keys. + the recompute reads it from the pre-split table the same way), exactly at + ``K = 1`` and at ``K > 1`` to the family weights' relative tolerance of + 1e-9 (the fold re-sums ``n`` copies of a clone's share, which drifts by + more ulps than the count's snap absorbs), every member of a family + carries ``w / n`` at stage time (its clone pair sum, since the clone runs + afterwards), the copies of each root are indexed ``1..n-1`` without gaps + and carry the root's count, no count is below one, and the stage's mass + record conserves the total. The recompute is repeated on permuted person + and household tables and must reproduce the divided set and the counts + exactly. A selected household light enough for a single copy leaves no + trace in the artifact, so only families are compared. Problems land under + ``support_split_*`` keys. """ from microcosm.build.uk_runtime.cgt_support import ( @@ -1044,9 +1047,26 @@ def recompute(person_t, household_t, weights_t): "extra": extra_families, } agreed = (recomputed_family & stored_family).to_numpy() - count_off = int( - (original["copies"].to_numpy()[agreed] != stored_count.to_numpy()[agreed]).sum() - ) + stored_agreed = stored_count.to_numpy()[agreed] + if clone_count == 1: + count_off = int((original["copies"].to_numpy()[agreed] != stored_agreed).sum()) + else: + # At K > 1 the fold re-sums n copies of a K-th share, which drifts by + # more ulps than the count's K-ulp snap absorbs, so an exact recount + # would flag a family the stage counted right (an exact multiple of 60 + # at K = 11, for one). The count is held to the family weights' + # tolerance instead: the stored n must lie between the counts at the + # folded weight times 1 - 1e-9 and 1 + 1e-9 (microcosm#931 review). + agreed_weights = pre_weights[agreed] + lowest, highest = ( + cgt_support_copy_counts( + agreed_weights * factor, + CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + clone_count=clone_count, + ) + for factor in (1.0 - 1e-9, 1.0 + 1e-9) + ) + count_off = int(((stored_agreed < lowest) | (stored_agreed > highest)).sum()) if count_off: problems["support_split_copies_stored"] = count_off From bad1f179a0866665d8f38dbcd35f09e2a38d7db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:50:29 +0000 Subject: [PATCH 019/108] Tidy the re-reviews of the copy-id decoders and the CGT copy count (#931) The re-reviews of da5a3502d and 9721448a9 found no must and no code should. This commit takes their four nits. - Bindings: the stochastic_assignment exemption of spi_support_channel now names the stage's transform, and the binding test checks that the transform names no drawing function, directly or through its module, as well as the drawing names the module imports. Before, a support-channel transform calling stage 14's employer pension draw, which spi_spine already imports, would have passed both binding tests. - The row-wise tool's --source-lineage-modulus help still described the spine's retired copy-id scheme. It now says spine inputs carry copy ids in the copy-id table's fixed fields. - Test comments on the CGT nesting scenario: the 9,000 household is the first candidate for the exact-total correction, not where it always lands. At 7 and 13 clones the shared fallback puts it on the last copy of the 7,620 family's last clone. At 13 clones an exact recount moves 12 of the 13 clones, not every one. - The identity tool's E8 docstring says copies_recomputed stays the exact recount, so at K > 1 it can exceed copies by one for each exact-multiple family while the count check passes. What moves: nothing. No runtime module changes, so no node key moves. Ruff is clean, the stochastic_assignment binding tests pass, and the re-review's mutant (the exempt transform naming the employer pension draw) now fails the binding test. Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/uk/test_uk_cgt_support.py | 9 ++++++--- .../tests/engine_free/uk/test_uk_graph.py | 14 ++++++++++---- .../uk/test_uk_identity_stability_receipts.py | 14 ++++++++++---- tools/build_uk_rowwise_dataset.py | 8 ++++---- tools/verify_uk_identity_stability.py | 6 +++++- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 74487511c9..59be3c5aba 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -1283,9 +1283,12 @@ def _leaves(value): # household is outside every pool. The synthetic joint asks band 0 for far # more than its pool, so the whole pool is selected at every clone count and # the selection nests exactly; band 37,700 asks for nothing, so its household -# takes the exact-total correction (on clone 0, the first heaviest), heavy -# enough (9,000) that the correction moves it far less than the weights' -# nesting tolerance. +# (9,000) is the first candidate for the exact-total correction (on clone 0, +# the first heaviest). At seven (and 13) clones its clone 0 has no +# representable correction, so the shared fallback puts it on the last copy +# of the 7,620 family's last clone, a few ulps off w / n and within the +# weights' nesting tolerance and the gate's; at one, three, 11 and 15 clones +# no correction is needed. NESTING_WEIGHTS = [ 900.0, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 323d90ede9..fe32e5a350 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -425,9 +425,11 @@ def changed_helper_bytes(path: Path) -> bytes: _STOCHASTIC_ASSIGNMENT = "microcosm.build.stochastic_assignment" #: Stages that import a function drawing through the keyed-draw helpers #: (``microcosm.build.stochastic_assignment``) but never call it, so need not -#: bind them: the drawing names each imports, and why it never calls them. -_STOCHASTIC_ASSIGNMENT_UNBOUND: dict[str, tuple[frozenset[str], str]] = { +#: bind them: the stage's transform, the drawing names its module imports, and +#: why the transform never calls them. +_STOCHASTIC_ASSIGNMENT_UNBOUND: dict[str, tuple[str, frozenset[str], str]] = { "spi_support_channel": ( + "UKSPISupportChannelStageTransform", frozenset( { "employer_pension_contributions.draw_employer_pension_contributions", @@ -627,9 +629,13 @@ def test_every_stage_that_draws_through_stochastic_assignment_binds_it() -> None assert reach["frs_hmrc_spine_leaves"] >= { "employer_pension_contributions.draw_employer_pension_contributions" } - # An exempt stage imports exactly the drawing names its reason covers. - for stage, (names, _) in _STOCHASTIC_ASSIGNMENT_UNBOUND.items(): + # An exempt stage imports exactly the drawing names its reason covers, and + # its transform names none of them, directly or through its module. + for stage, (transform, names, _) in _STOCHASTIC_ASSIGNMENT_UNBOUND.items(): assert reach.get(stage) == names, stage + module = graph_kernels._stage_module(stage).__name__ + assert transform in _draw_facts(module).references, stage + assert transform not in _drawing_names([module])[module], stage def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 0efb08503c..28296881d3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1073,8 +1073,12 @@ def test_an_artifact_without_the_split_layer_is_refused(self) -> None: #: 1,800, 4,020 and 7,620: 15, 30, 67 and 127 copies), a weight one ulp below #: 300 (5 copies), a light one (40, uncopied) and a non-multiple (1,234.5, id #: 7, 21 copies); band 0's pool is exhausted at every clone count, band -#: 37,700 asks for nothing, so its household (9,000) takes the exact-total -#: correction, and the zero-weight household is outside every pool. +#: 37,700 asks for nothing, so its household (9,000) is the first candidate +#: for the exact-total correction, and the zero-weight household is outside +#: every pool. At 7 and 13 clones that household's clone 0 has no +#: representable correction, so the shared fallback puts it on the last copy +#: of the 7,620 family's last clone, a few ulps off w / n and within the +#: 1e-12 tolerances; at 1, 3, 11 and 15 clones no correction is needed. _NESTING_SCENARIO = { "weights": [ 900.0, @@ -1217,8 +1221,10 @@ def test_exact_multiples_of_60_receipt_green_at_any_clone_count( ) -> None: # At 11 and 13 clones the fold re-sums the 7,620 family's 127 copies # to a weight whose K x w / 60 sits more ulps above 127 than the - # count's snap absorbs, so an exact recount would count 128 in every - # clone; the tolerance accepts the stage's 127. + # count's snap absorbs, so an exact recount would count 128 in all 11 + # clones at 11 and in 12 of the 13 at 13 (the clone whose last copy + # took the exact-total correction re-sums to 127); the tolerance + # accepts the stage's 127. one_split, one = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) split, many = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 0a60c5db29..5c89130a2f 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -217,10 +217,10 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help=( "Pool inputs only: derive pool_source_household_id before cloning " "from household_id = tier * 10**8 + base, leaving the immediate " - "source_household_id untouched. Spine inputs instead use sernum + " - "1{spi} * 10**d + 1{cgt_clone} * 10**(d+1) + 1{band_donor} * " - "10**(d+2) and carry authoritative explicit lineage columns, so " - "the modulus is refused for them. Also refused when the pool " + "source_household_id untouched. Spine inputs instead carry copy " + "ids in the copy-id table's fixed fields (microcosm#931) and " + "authoritative explicit lineage columns, so the modulus is " + "refused for them. Also refused when the pool " "column exists or the mapping would be an identity." ), ) diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index ffa063b0dd..9ce81b7e27 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -847,7 +847,11 @@ def _e8_support_split( the recompute reads it from the pre-split table the same way), exactly at ``K = 1`` and at ``K > 1`` to the family weights' relative tolerance of 1e-9 (the fold re-sums ``n`` copies of a clone's share, which drifts by - more ulps than the count's snap absorbs), every member of a family + more ulps than the count's snap absorbs; the receipt's + ``copies_recomputed`` stays the exact recount, so at ``K > 1`` it can + exceed ``copies`` by one for each family at an exact multiple whose + re-summed weight lands above it, while the count check passes), every + member of a family carries ``w / n`` at stage time (its clone pair sum, since the clone runs afterwards), the copies of each root are indexed ``1..n-1`` without gaps and carry the root's count, no count is below one, and the stage's mass From 3903903fb010e86c90cd0a136150d067887dd05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:00:26 +0000 Subject: [PATCH 020/108] Say where the held land rows are measured (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The land changelog said the two held rows "are measured on every evaluation", and 161c77745's test was named for that. That phrase is the exclusions register's convention, and the register's entries keep it, but no build step reports a held row's estimate. As María ruled on 8 Nov, the changelog and the test now say what happens: the rows stay on the run's compiled register, and a tool that scores the un-excluded register, such as tools/evaluate_uk_incumbent_surface.py, measures them, as it does every held row. - The changelog fragment's sentence is reworded. - test_the_two_land_diagnostics_are_measured_but_not_fitted is renamed test_the_two_land_diagnostics_leave_the_solve_but_stay_on_the_compiled_register, and its docstring and the scoring comment say the same. What moves: nothing. Ruff is clean and the renamed test passes. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-land-rows-diagnostic.changed.md | 2 +- .../engine_free/uk/test_uk_measure_simulation.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/changelog.d/931-uk-land-rows-diagnostic.changed.md b/changelog.d/931-uk-land-rows-diagnostic.changed.md index a5c6e0a5da..0a7f853120 100644 --- a/changelog.d/931-uk-land-rows-diagnostic.changed.md +++ b/changelog.d/931-uk-land-rows-diagnostic.changed.md @@ -1,4 +1,4 @@ -The UK calibration holds two of the three ONS national balance sheet land rows out of the solve as measured diagnostics (#931; María's ruling of 6 November 2026, workstream 2 of microcosm#1521). `ons.land.corporate_land_value` (E43B, non-financial corporations' land) reads policyengine-uk's `corporate_land_value`, which hands out the engine's own aggregate by each household's share of the weighted corporate-sector wealth: a released file meets that aggregate at any weights, and the solve, fitting the column measured at the starting weights, pinned only the weighted mass of the WAS corporate and private pension wealth, not land. `ons.land.land_value` (E44R, the total economy) also counts the land of financial corporations, government and NPISH, which neither the household row (E44N) nor the corporate row covers, while the engine's `land_value` is the sum of the two columns those rows read, so beside them it asked for £323,750m that no weights could supply. Both are measured on every evaluation, under reviewed exclusions to 2027-02-06. `ons.land.household_land_value` stays bound, and its binding note now names the households sector, the one E44N covers, not households and NPISH; the two held rows' binding notes record the hold. +The UK calibration holds two of the three ONS national balance sheet land rows out of the solve as measured diagnostics (#931; María's ruling of 6 November 2026, workstream 2 of microcosm#1521). `ons.land.corporate_land_value` (E43B, non-financial corporations' land) reads policyengine-uk's `corporate_land_value`, which hands out the engine's own aggregate by each household's share of the weighted corporate-sector wealth: a released file meets that aggregate at any weights, and the solve, fitting the column measured at the starting weights, pinned only the weighted mass of the WAS corporate and private pension wealth, not land. `ons.land.land_value` (E44R, the total economy) also counts the land of financial corporations, government and NPISH, which neither the household row (E44N) nor the corporate row covers, while the engine's `land_value` is the sum of the two columns those rows read, so beside them it asked for £323,750m that no weights could supply. Both are held under reviewed exclusions to 2027-02-06 and stay on the run's compiled register. No build step reports a held row's estimate; a tool that scores the un-excluded register, such as `tools/evaluate_uk_incumbent_surface.py`, measures them, as it does every held row. `ons.land.household_land_value` stays bound, and its binding note now names the households sector, the one E44N covers, not households and NPISH; the two held rows' binding notes record the hold. - **What moves in the solve.** Two rows leave it on both lines. Both lines give each target family an equal share of the objective and split it evenly over the family's bound rows (on the dense line, within each grain, where the land rows are amounts), so `ons.land.household_land_value`, now the `ons_land` family's only bound row, carries the family's whole share, three times its loss weight before. No other row's weight changes, since the family keeps a row. The calibrated weights move slightly through both effects, which a calibrate-only rerun on the move's spine is to measure together. The released corporate land value still equals the engine's aggregate, and the released land value moves only through household land. - **What else moves.** The national frozen scoring register (`national_target_registry.json`, the approved register) loses the two rows, so a national run given a frozen register derived before this change refuses, as with `obr.ni_employer`; the dense line checks its frozen register against the reconciled register, which keeps them. The incumbent and release-cut scores, `uk_target_fit` and `calibration_diagnostics.json` read only the rows the solve binds, so the two rows leave them. Neither line resolves `land_value` or `corporate_land_value` any more: the dense measure receipt's `national_inputs` falls by two, and a per-block dense run's `block_sensitivity.present_in_this_run` is empty. The dense release lists the two exclusions among its limitations (`measure_exclusion:`, with the first 300 characters of each reason). The edited files are UK country-spec resources, so the national and dense target, measure and solve nodes re-key; spine nodes, `source_stages.json` and the H2 spine fixture do not move. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py index 4a377de01c..af65a40f40 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_measure_simulation.py @@ -1256,10 +1256,12 @@ def published_millions(name: str) -> int: ) in held, name -def test_the_two_land_diagnostics_are_measured_but_not_fitted(): +def test_the_two_land_diagnostics_leave_the_solve_but_stay_on_the_compiled_register(): """The packaged register's two land entries take their rows out of the - national solve and leave them measured (the #931 move, María's ruling of - 2026-11-06). + national solve and keep them on the compiled register (the #931 move, + María's ruling of 2026-11-06). No build step reports a held row's + estimate; a tool that scores the un-excluded register, such as + tools/evaluate_uk_incumbent_surface.py, measures them. The test applies the register's land entries to the compiled rows as both target loaders (load_uk_national_target_inputs and @@ -1431,9 +1433,9 @@ def test_the_two_land_diagnostics_are_measured_but_not_fitted(): ) assert problem.bindings["target_loss_weights"] == pytest.approx([0.5, 0.5]) - # Measured: the compiled register materializes on the same frame, and - # scoring it gives every row, the two held land rows included, its - # weighted estimate. + # Scoreable: the compiled register materializes on the same frame, and + # scoring it, as a tool that scores the un-excluded register does, gives + # every row, the two held land rows included, its weighted estimate. adapter = CalibrationFrameAdapter(frame) materialized = materialize_uk_ledger_targets( adapter, compiled, period=2025, band_edge_registry=compiled From 06b1aa3201a5b8b59f2dc8f0e2de38547bc13e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:25:32 +0000 Subject: [PATCH 021/108] Retire the unused legacy UK national sampler (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As María ruled on 8 Nov, national_sampling's compact sampler, sample_uk_national_frame, is retired with its family decoder, uk_source_family_units. No build or tool has called the sampler since 8adc836b9 (#762 PR A) moved the row-wise candidate onto the spine sampler. The decoder re-derived retired id arithmetic (a clone multiplier of ten to the digit count of the largest canonical id, and max + 1 SPI and capital-gains offsets), which kept the module on the copy-id tests' inline-rule allowlist. - national_sampling.py loses the two functions, their __all__ entries and the helpers only they used: the post-sample SPI replacement quota check, the required-column list and the clone-index column alias. The module docstring now describes the spine sampler. The spine sampler, the rung tokens, the default seed and every name other modules read from the module are unchanged. - test_uk_national_sampling.py drops the ten tests of the retired functions and their compact-frame builder. The numeric-region refusal, which only those tests covered, is now tested on the spine sampler. - test_uk_copy_id_table.py: national_sampling leaves _INLINE_DIGIT_COUNT_MODULES, so the inline-rule guard now admits only rowwise_geography and copy_id_table. - experiments/901-uk-main-rebase-receipts.md cited one of the removed tests in its row 4; the row gains a dated note. - A removed changelog fragment. What moves: no value. _PopulationKernel hashes national_sampling, so uk.full.sample, uk.full.normalize, uk.full.expand, uk.full.locations, uk.full.geography_mapping, uk.full.pool and uk.full.geography_gate re-key, and with them every node downstream of uk.full.sample. No other UK kernel binds the module, so no spine node re-keys. No generated file moves: the source_stages projection and coverage manifest checks pass, and the H2 fixture holds no code hashes. Tests: test_uk_national_sampling, test_uk_copy_id_table and test_uk_runtime_exports pass (57 tests). An AST scan finds no tracked Python file that names the two functions, and the inline-rule detector flags the module at the parent commit but not here. A permissive region check fails the new spine test. Ruff is clean. Co-Authored-By: Claude Opus 5.5 --- ...-retire-legacy-national-sampler.removed.md | 1 + experiments/901-uk-main-rebase-receipts.md | 5 +- .../build/uk_runtime/national_sampling.py | 311 ++---------------- .../engine_free/uk/test_uk_copy_id_table.py | 5 - .../uk/test_uk_national_sampling.py | 297 ++--------------- 5 files changed, 46 insertions(+), 573 deletions(-) create mode 100644 changelog.d/931-retire-legacy-national-sampler.removed.md diff --git a/changelog.d/931-retire-legacy-national-sampler.removed.md b/changelog.d/931-retire-legacy-national-sampler.removed.md new file mode 100644 index 0000000000..3642de9de5 --- /dev/null +++ b/changelog.d/931-retire-legacy-national-sampler.removed.md @@ -0,0 +1 @@ +Retire the UK national compact sampler, `sample_uk_national_frame`, and its source-family decoder, `uk_source_family_units`, from `microcosm.build.uk_runtime.national_sampling` (#931). The sampler drew rungs from the certified national compact. The decoder recovered each derivative row's source from retired id arithmetic: a clone multiplier of ten to the digit count of the largest canonical id, and SPI and capital-gains offsets of the largest raw and pre-capital-gains household ids plus one. No build or tool has called either since 8adc836b9 (#762) moved the row-wise candidate onto the spine sampler. Their private helpers go with them: the post-sample SPI replacement quota check, the required-column list and the clone-index column name. The module also leaves the copy-id tests' inline-rule allowlist, which now admits only `rowwise_geography` (the dense line's multiplier) and `copy_id_table` (its field spans). The spine sampler (`sample_uk_spine_frame`, `uk_spine_source_family_units`), the rung tokens and the default sampling seed are unchanged. The tests of the retired functions are removed, and the numeric-region refusal they covered is now tested on the spine sampler. No value moves. The full build's population nodes that hash the module (`uk.full.sample`, `uk.full.normalize`, `uk.full.expand`, `uk.full.locations`, `uk.full.geography_mapping`, `uk.full.pool` and `uk.full.geography_gate`) re-key, and with them every node downstream of `uk.full.sample`. No spine node re-keys, and no generated file moves. diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index 3f995ce2c9..92310d0600 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -131,7 +131,10 @@ closed on 2026-09-28 by restoring the tool's behaviour on the driver. 4. `candidate_f100_does_not_call_any_sampler`: covered by `test_uk_national_sampling.py::test_full_fraction_is_a_structural_no_op` and `::test_source_sampling_cannot_be_reapplied_as_pool_sampling`; the compact national sampler - is not reachable from the graph's population node. + is not reachable from the graph's population node. (Note of 2026-11-08: microcosm#931 + retired the compact national sampler, `sample_uk_national_frame`, which no build or tool + had called since 8adc836b9, together with that test. The second test and + `uk.full.sample`'s f100 branch, which calls no sampler, remain.) 5. `candidate_engine_surface_reuses_one_resolver`: covered by `test_uk_full_measure.py::test_full_measure_reuses_one_resolver` (direct port). 6. `candidate_engine_surface_resolves_real_per_clone_blocks`: covered by diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py index 67a30316ca..a90978b607 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py @@ -1,34 +1,23 @@ -"""Scale-ladder sampling policy for the UK national build (#627). - -The UK samples a mid-pipeline artifact: the certified compact already carries -geography/CG clone families (``clone_index`` reverses to the canonical -``clone_index == 0`` surface by ID arithmetic), SPI-synthetic and -capital-gains derivative rows that reverse **to the raw FRS surface** by -max-derived offsets (``max(canonical raw ids) + 1``), and the zero-weight SPI -dead channel whose rebuild enforces a per-stratum ``#base >= #dead`` quota. -A uniform household draw would break all three, so the UK policy over the -shared :mod:`microcosm.build.frame_sampling` core is: - -- **The sampling unit is the source FRS family**: the raw canonical - household plus every SPI/CG derivative and every geography clone that - reverses onto it through both ID arithmetics. A drawn family therefore - keeps clone reversal *and* the SPI/CG source reversal intact — a derived - row never survives without the raw row it reverses to. -- **The reversal constants are pinned by forced retention.** The stage - fence re-derives the clone multiplier (``10 ** len(str(max canonical - id))``) and the SPI/CG offsets (``max(canonical raw id) + 1``, - ``max(canonical pre-CG id) + 1``) from surviving data, so the families - carrying each argmax id — canonical, raw, and pre-CG, household and - person — are always retained and every constant is stable. -- **The draw is stratified by the raw canonical household's region.** - Channel flags vary *within* a source family (raw + SPI + CG rows travel - together), so they are not strata; the per-cell SPI quota is preserved - structurally instead — a dead row's in-cell base source is in its own - family — and a post-sample check re-asserts the full per-cell quota, - failing closed on any irregular input. +"""Scale-ladder sampling for the UK spine (#627). + +The UK full build samples the Microcosm spine at its ``uk.full.sample`` node, +before geographic expansion. Every derivative row on the spine names its raw +FRS household in ``source_household_id``, so the UK policy over the shared +:mod:`microcosm.build.frame_sampling` core is: + +- **The sampling unit is the source family**: the raw FRS household and + every row whose ``source_household_id`` names it. A drawn family is kept + whole, so a derived row never survives without its raw row; the families + are derived again on the sample, and a family left incomplete is refused. +- **The draw is stratified by the raw rows' region.** Derivative flags and + derivative rows' own regions never split a family across strata, and a + family whose raw rows span more than one region is refused. - **Sampled mass is renormalized to the full-source household total** (the - US stacked semantics #627 names), so the anchor population and the SPI - 50/50 mass-share allocation behave identically at every rung. + US stacked semantics #627 names). + +A spine without lineage columns is sampled household by household, +stratified by region. The module also holds the rung tokens and the default +sampling seed that the UK builds share. """ from __future__ import annotations @@ -54,10 +43,8 @@ __all__ = [ "UK_SAMPLE_RUNG_TOKENS", "UK_SAMPLE_SEED_DEFAULT", - "sample_uk_national_frame", "sample_uk_spine_frame", "uk_spine_source_family_units", - "uk_source_family_units", ] #: The #624 scale-ladder rungs and their identity tokens, shared with the US @@ -74,20 +61,12 @@ UK_SAMPLE_SEED_DEFAULT = 578 ( - _CLONE_INDEX_COLUMN, + _, _CAPITAL_GAINS_CLONE_COLUMN, _REGION_COLUMN, ) = SPI_REPLACEMENT_STRATA_COLUMNS -def _required_household_columns() -> tuple[str, ...]: - return ( - "household_id", - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - *SPI_REPLACEMENT_STRATA_COLUMNS, - ) - - def _strict_bool(values: pd.Series, *, label: str) -> np.ndarray: if values.isna().any(): raise ValueError(f"UK sample: {label} contains missing values.") @@ -299,253 +278,3 @@ def sample_uk_spine_frame( "strata_count": int(len(np.unique(strata))), "receipt": dict(core_receipt), } - - -def uk_source_family_units( - frame: Frame, -) -> tuple[np.ndarray, tuple[int, ...], int]: - """Per-household-row source-FRS-family key, mirroring the stage fence. - - Reproduces ``_resolve_candidate_lineage``'s two-layer arithmetic: the - clone multiplier is ``10 ** max(1, len(str(canonical_max)))`` over the - ``clone_index == 0`` household **and** person ids; the SPI and CG - household offsets are ``max(canonical raw household id) + 1`` and - ``max(canonical pre-CG household id) + 1``; and every row's family key is - - ``household_id - clone_index*multiplier - spi*spi_offset - cg*cg_offset`` - - which must land on an existing canonical raw FRS household. Fails closed - before any draw on household-level rows the stage fence would refuse; - person-level lineage defects are outside this pre-check and surface at - stage one, where the fence stays strict. - - Returns: - The per-household-row family keys, the forced family keys (the - families carrying every argmax id the fence's constants are derived - from — retaining them pins the multiplier's digit count and both - offsets exactly), and the clone multiplier. - """ - - household = frame.table("household") - missing = sorted(set(_required_household_columns()) - set(household.columns)) - if missing: - raise ValueError( - "UK sample requires the certified compact's lineage and channel " - f"columns; household is missing {missing}." - ) - person = frame.table("person") - household_ids = _int_column(household["household_id"], label="household_id") - clone_index = _int_column(household[_CLONE_INDEX_COLUMN], label=_CLONE_INDEX_COLUMN) - if (clone_index < 0).any(): - raise ValueError("UK sample: clone_index must be non-negative.") - spi_flag = _strict_bool( - household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN], - label=HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - ) - cg_flag = _strict_bool( - household[_CAPITAL_GAINS_CLONE_COLUMN], - label=_CAPITAL_GAINS_CLONE_COLUMN, - ) - canonical_households = clone_index == 0 - if not canonical_households.any(): - raise ValueError("UK sample requires clone_index=0 canonical rows.") - - person_ids = _int_column(person["person_id"], label="person_id") - person_household = _int_column( - person["person_household_id"], label="person_household_id" - ) - household_position = pd.Series(np.arange(len(household_ids)), index=household_ids) - person_household_position = household_position.reindex(person_household) - if person_household_position.isna().any(): - raise ValueError( - "UK sample: person_household_id cannot map every person to a household row." - ) - person_positions = person_household_position.to_numpy(dtype=np.int64) - person_clone_index = clone_index[person_positions] - person_spi = spi_flag[person_positions] - person_cg = cg_flag[person_positions] - canonical_people = person_clone_index == 0 - if not canonical_people.any(): - raise ValueError("UK sample requires clone_index=0 person rows.") - - canonical_max = max( - int(household_ids[canonical_households].max()), - int(person_ids[canonical_people].max()), - ) - multiplier = 10 ** max(1, len(str(canonical_max))) - clone_reversed = household_ids - clone_index * multiplier - if (clone_reversed <= 0).any(): - raise ValueError( - "UK sample: clone reversal produced non-positive canonical ids; " - "the input's clone lineage does not match the stage fence's " - "arithmetic." - ) - canonical_set = household_ids[canonical_households] - unknown_clones = np.setdiff1d(np.unique(clone_reversed), canonical_set) - if len(unknown_clones): - raise ValueError( - f"UK sample: {len(unknown_clones)} clone household(s) do not " - "reverse to a clone_index=0 canonical row; refusing to sample an " - "input the stage fence would reject." - ) - - raw_households = canonical_households & ~spi_flag & ~cg_flag - pre_cg_households = canonical_households & ~cg_flag - if not raw_households.any(): - raise ValueError("UK sample requires canonical raw FRS households.") - spi_offset = int(household_ids[raw_households].max()) + 1 - cg_offset = int(household_ids[pre_cg_households].max()) + 1 - units = ( - clone_reversed - - spi_flag.astype(np.int64) * spi_offset - - cg_flag.astype(np.int64) * cg_offset - ) - raw_set = frozenset(int(value) for value in household_ids[raw_households]) - if (units <= 0).any() or not set(int(v) for v in np.unique(units)) <= raw_set: - bad = len(set(int(v) for v in np.unique(units)) - raw_set) - raise ValueError( - f"UK sample: {bad} household(s) do not reverse to the raw FRS " - "surface; refusing to sample an input the stage fence would " - "reject." - ) - - def _unit_of_max_household(mask: np.ndarray) -> int: - position = int(np.argmax(np.where(mask, household_ids, -1))) - return int(units[position]) - - def _unit_of_max_person(mask: np.ndarray) -> int: - position = int(np.argmax(np.where(mask, person_ids, -1))) - return int(units[person_positions[position]]) - - raw_people = canonical_people & ~person_spi & ~person_cg - pre_cg_people = canonical_people & ~person_cg - if not raw_people.any(): - raise ValueError("UK sample requires canonical raw FRS person rows.") - forced = tuple( - sorted( - { - # Clone-multiplier pins (digit count of the canonical max). - _unit_of_max_household(canonical_households), - _unit_of_max_person(canonical_people), - # SPI-offset pins (max canonical raw id, exactly). - _unit_of_max_household(raw_households), - _unit_of_max_person(raw_people), - # CG-offset pins (max canonical pre-CG id, exactly). - _unit_of_max_household(pre_cg_households), - _unit_of_max_person(pre_cg_people), - } - ) - ) - return units, forced, multiplier - - -def _assert_spi_replacement_quota(household: pd.DataFrame) -> None: - """Re-assert the SPI rebuild's per-cell ``#base >= #dead`` on the sample. - - Source-family sampling preserves the inequality structurally when every - dead row's source lives in its own cell; this check converts that - assumption into a fail-closed runtime receipt, using the same stratum - cells the rebuild's ``_sample_replacement_household_ids`` groups by. - """ - - synthetic = _strict_bool( - household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN], - label=HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - ) - dead = household.loc[synthetic] - if dead.empty: - return - strata = list(SPI_REPLACEMENT_STRATA_COLUMNS) - base = household.loc[~synthetic] - dead_counts = dead.groupby(strata, sort=True, dropna=False).size() - base_counts = base.groupby(strata, sort=True, dropna=False).size() - for key, quota in dead_counts.items(): - capacity = int(base_counts.get(key, 0)) - if capacity < int(quota): - raise ValueError( - "UK sample violates the SPI replacement quota for stratum " - f"{key!r}: need {int(quota)} base household(s), found " - f"{capacity}. Source-family sampling preserves this " - "inequality when dead rows share their source's cell; an " - "irregular structure in the input is the likely cause." - ) - - -def sample_uk_national_frame( - frame: Frame, - *, - fraction: float, - seed: int, -) -> tuple[Frame, dict[str, object]]: - """Draw one seeded UK rung sample and renormalize it to full mass. - - Runs on the loaded certified compact **before** provenance binding, so - the certified-candidate fence attests the frame the stages actually - consume. The result satisfies :func:`validate_uk_national_frame`: the - kernel mints the renormalization :class:`MassChangeRecord`, and the - exported ``household_weight`` column is refreshed from the typed vector. - - Returns: - The sampled frame and the shared sampling receipt extended with the - normalization fields and the UK policy block. - """ - - validate_sample_fraction(fraction, label="UK sample") - validate_sample_seed(seed, label="UK sample") - validate_uk_national_frame(frame) - - household = frame.table("household") - units, forced, multiplier = uk_source_family_units(frame) - household_ids = _int_column(household["household_id"], label="household_id") - region_values = _str_column(household[_REGION_COLUMN], label=_REGION_COLUMN) - region_by_household = dict( - zip( - household_ids.tolist(), - region_values.tolist(), - strict=True, - ) - ) - # The stratum is the raw canonical household's region: clone-0 quota - # cells vary by family region, so proportionality must hold within each - # canonical region. Channel flags vary within a source family and are - # deliberately not strata — derived rows travel with their source. - strata = np.asarray( - [f"region={region_by_household[int(unit)]}" for unit in units], - dtype=object, - ) - sampled, receipt = sample_frame_households( - frame, - fraction=fraction, - seed=seed, - source_name="UK national", - unit_ids=units, - unit_strata=strata, - forced_unit_ids=forced, - unit_noun="source family", - floor_context="the UK national build", - ) - if sampled is not frame: - full_mass = float(receipt["incoming_household_mass"]) - sampled, factor = normalize_sampled_household_mass( - sampled, target_mass=full_mass, source_name="UK national" - ) - receipt["normalization_factor"] = factor - receipt["normalized_household_mass"] = float( - sampled.weights_for("household").total - ) - _assert_spi_replacement_quota(sampled.table("household")) - validate_uk_national_frame(sampled) - receipt["uk_policy"] = { - "sampling_unit": "source_frs_family", - "strata_columns": [ - f"{_REGION_COLUMN} (of the raw clone_index=0 canonical row)" - ], - "clone_multiplier": multiplier, - "forced_retention": ( - "families carrying the argmax canonical, raw, and pre-CG " - "household and person ids (clone-multiplier and SPI/CG offset " - "pins)" - ), - "spi_replacement_quota_checked": True, - } - return sampled, receipt diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py index b47733453a..92964537e6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -754,11 +754,6 @@ def test_only_the_dense_line_names_the_retired_id_rule() -> None: "packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py": ( "sizes each declared field's span from its ordinal bound" ), - "packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py": ( - "uk_source_family_units, the legacy candidate decoder, re-derives the " - "retired stage fence's clone multiplier; whether it learns the table " - "is open (microcosm#931)" - ), } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py index 565b9bc999..b8eb24e138 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py @@ -1,4 +1,4 @@ -"""UK scale-ladder sampling policy (#627): source families, strata, quota.""" +"""UK spine sampling policy (#627): source families, strata, normalization.""" from __future__ import annotations @@ -6,142 +6,13 @@ import pandas as pd import pytest -from microcosm.build.uk_runtime.national_frame import ( - uk_national_frame, - validate_uk_national_frame, -) +from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.national_sampling import ( - sample_uk_national_frame, sample_uk_spine_frame, - uk_source_family_units, uk_spine_source_family_units, ) from microcosm.frame import Frame -_RAW = ( - {"id": 101, "region": "london", "weight": 10.0}, - {"id": 102, "region": "north", "weight": 10.0}, - {"id": 103, "region": "london", "weight": 10.0}, - {"id": 104, "region": "north", "weight": 10.0}, - {"id": 105, "region": "london", "weight": 10.0}, - {"id": 106, "region": "north", "weight": 10.0}, -) - - -def _source_family_frame( - *, - spi_of: tuple[int, ...] = (103, 104), - cg_of: tuple[int, ...] = (101,), - spi_region_override: str | None = None, - corrupt_spi_household: int | None = None, -) -> Frame: - """Build the two-layer synthetic compact the sampler's arithmetic expects. - - Raw canonical households get one person each (``household_id * 1000 + 1``, - satisfying the stage fence's ``person // 1000 == household`` invariant so - these frames are fence-valid, not just sampler-valid); - SPI/CG derivatives are constructed with the fence's max-derived offsets - (``max(canonical raw id) + 1`` / ``max(canonical pre-CG id) + 1``); every - canonical row then gets one geography clone at ``+ clone_multiplier``. - """ - - rows: list[dict[str, object]] = [] - for family in _RAW: - rows.append( - { - "household_id": int(family["id"]), - "person_id": int(family["id"]) * 1000 + 1, - "region": family["region"], - "spi": False, - "cg": False, - "weight": float(family["weight"]), - } - ) - raw_by_id = {row["household_id"]: row for row in rows} - spi_household_offset = max(int(f["id"]) for f in _RAW) + 1 - spi_person_offset = max(int(f["id"]) * 1000 + 1 for f in _RAW) + 1 - for source_id in spi_of: - source = raw_by_id[source_id] - rows.append( - { - "household_id": source_id + spi_household_offset, - "person_id": int(source["person_id"]) + spi_person_offset, - "region": ( - spi_region_override - if spi_region_override is not None - else source["region"] - ), - "spi": True, - "cg": False, - "weight": 0.0, - } - ) - cg_household_offset = max(int(r["household_id"]) for r in rows) + 1 - cg_person_offset = max(int(r["person_id"]) for r in rows) + 1 - for source_id in cg_of: - source = raw_by_id[source_id] - rows.append( - { - "household_id": source_id + cg_household_offset, - "person_id": int(source["person_id"]) + cg_person_offset, - "region": source["region"], - "spi": False, - "cg": True, - "weight": float(source["weight"]), - } - ) - if corrupt_spi_household is not None: - for row in rows: - if row["spi"]: - row["household_id"] = corrupt_spi_household - break - canonical_max = max( - max(int(r["household_id"]) for r in rows), - max(int(r["person_id"]) for r in rows), - ) - multiplier = 10 ** max(1, len(str(canonical_max))) - clone_rows = [ - { - **row, - "household_id": int(row["household_id"]) + multiplier, - "person_id": int(row["person_id"]) + multiplier, - "region": "scotland", - } - for row in rows - ] - all_rows = sorted(rows + clone_rows, key=lambda row: row["household_id"]) - household_ids = np.asarray( - [row["household_id"] for row in all_rows], dtype=np.int64 - ) - household = pd.DataFrame( - { - "household_id": household_ids, - "clone_index": [ - 1 if int(row["household_id"]) > multiplier else 0 for row in all_rows - ], - "region": [row["region"] for row in all_rows], - "household_is_spi_synthetic": [row["spi"] for row in all_rows], - "household_is_capital_gains_clone": [row["cg"] for row in all_rows], - "household_weight": [row["weight"] for row in all_rows], - } - ) - person = pd.DataFrame( - { - "person_id": np.asarray( - [row["person_id"] for row in all_rows], dtype=np.int64 - ), - "person_benunit_id": household_ids + 1_000_000_000, - "person_household_id": household_ids, - } - ) - benunit = pd.DataFrame({"benunit_id": household_ids + 1_000_000_000}) - return uk_national_frame( - person=person, - benunit=benunit, - household=household, - time_period="2023", - ) - def _spine_family_frame( *, @@ -278,151 +149,6 @@ def _spine_family_frame( ) -def _units_by_household(frame: Frame) -> dict[int, int]: - units, _forced, _multiplier = uk_source_family_units(frame) - household_ids = frame.table("household")["household_id"].to_numpy() - return dict(zip(household_ids.tolist(), units.tolist(), strict=True)) - - -def test_source_units_mirror_the_stage_fence_arithmetic() -> None: - frame = _source_family_frame() - units, forced, multiplier = uk_source_family_units(frame) - mapping = _units_by_household(frame) - # SPI derivatives reverse to their raw sources through max(raw)+1 = 107. - assert mapping[210] == 103 - assert mapping[211] == 104 - # The CG derivative reverses through max(pre-CG)+1 = 212. - assert mapping[313] == 101 - # Clones reverse through the multiplier first. - assert mapping[multiplier + 210] == 103 - assert mapping[multiplier + 313] == 101 - assert set(units.tolist()) == {101, 102, 103, 104, 105, 106} - # Forced pins: the argmax canonical ids live in the CG derivative of - # family 101, the raw argmaxes in 106, the pre-CG argmaxes in 104. - assert forced == (101, 104, 106) - - -def test_sampling_keeps_source_families_whole() -> None: - frame = _source_family_frame() - full_mapping = _units_by_household(frame) - rows_by_unit: dict[int, set[int]] = {} - for household_id, unit in full_mapping.items(): - rows_by_unit.setdefault(unit, set()).add(household_id) - for seed in (0, 1, 7, 42): - sampled, receipt = sample_uk_national_frame(frame, fraction=0.5, seed=seed) - sampled_ids = set(sampled.table("household")["household_id"].tolist()) - sampled_units = {full_mapping[value] for value in sampled_ids} - for unit in sampled_units: - assert rows_by_unit[unit] <= sampled_ids - # Forced retention keeps every pin family at every seed. - assert {101, 104, 106} <= sampled_units - assert receipt["uk_policy"]["sampling_unit"] == "source_frs_family" - - -def test_derived_rows_never_survive_without_their_source() -> None: - """Regression for the smoke-rung failure: SPI/CG ids must keep reversing. - - The first credentialed 1% run failed stage one with "Candidate - SPI/capital-gains person IDs do not reverse to the raw FRS surface" - because clone-family units let a derivative survive without its raw - source. Source-family units make that structurally impossible. - """ - - frame = _source_family_frame() - full_mapping = _units_by_household(frame) - for seed in (0, 3, 11): - sampled, _receipt = sample_uk_national_frame(frame, fraction=0.5, seed=seed) - household = sampled.table("household") - sampled_ids = set(household["household_id"].tolist()) - derived = household.loc[ - household["household_is_spi_synthetic"] - | household["household_is_capital_gains_clone"] - ] - for household_id in derived["household_id"].tolist(): - assert full_mapping[int(household_id)] in sampled_ids - - -def test_sampled_frame_is_normalized_refreshed_and_valid() -> None: - frame = _source_family_frame() - full_mass = float(frame.weights_for("household").total) - sampled, receipt = sample_uk_national_frame(frame, fraction=0.5, seed=3) - validate_uk_national_frame(sampled) - weights = sampled.weights_for("household") - assert float(weights.total) == pytest.approx(full_mass) - assert "household_weight" not in sampled.table("household") - record = sampled.mass_log[-1] - assert record.entity == "household" - assert "composition-preserving" in record.reason - assert receipt["normalized_household_mass"] == pytest.approx(full_mass) - - -def test_numeric_region_codes_are_refused() -> None: - """Region strata labels are formatted values, so an integer code in one - build and a float code in another would silently split a stratum; the - sampler accepts string regions only.""" - - frame = _source_family_frame() - household = frame.table("household").copy() - household["region"] = np.arange(len(household)) % 3 - numeric = uk_national_frame( - person=frame.table("person"), - benunit=frame.table("benunit"), - household=household, - time_period="2023", - household_weights=frame.weights_for("household").values, - ) - with pytest.raises(ValueError, match="must contain non-empty strings"): - sample_uk_national_frame(numeric, fraction=0.5, seed=1) - - -def test_strata_are_the_raw_canonical_regions() -> None: - frame = _source_family_frame() - _, receipt = sample_uk_national_frame(frame, fraction=0.5, seed=11) - strata = receipt["strata"] - assert set(strata) == {"region=london", "region=north"} - assert strata["region=london"]["eligible_units"] == 3 - assert strata["region=london"]["requested_units"] == 1 - assert strata["region=north"]["eligible_units"] == 3 - assert strata["region=north"]["requested_units"] == 1 - - -def test_spi_quota_violation_fails_closed() -> None: - # An SPI derivative living in a region its source does not: the sampled - # artifact carries a clone-0 wales cell with dead > base at any draw. - frame = _source_family_frame(spi_region_override="wales") - with pytest.raises(ValueError, match="SPI replacement quota"): - sample_uk_national_frame(frame, fraction=0.5, seed=5) - - -def test_non_reversing_derivative_fails_closed() -> None: - # Household 250 - 107 = 143 is not a raw canonical id. - frame = _source_family_frame(corrupt_spi_household=250) - with pytest.raises(ValueError, match="do not reverse to the raw FRS"): - uk_source_family_units(frame) - - -def test_missing_lineage_columns_fail_closed() -> None: - frame = _source_family_frame() - household = frame.table("household").drop(columns=["clone_index"]) - stripped = uk_national_frame( - person=frame.table("person"), - benunit=frame.table("benunit"), - household=household, - time_period="2023", - household_weights=frame.weights_for("household").values, - ) - with pytest.raises(ValueError, match="missing \\['clone_index'\\]"): - sample_uk_national_frame(stripped, fraction=0.5, seed=1) - - -def test_full_fraction_is_a_structural_no_op() -> None: - frame = _source_family_frame() - sampled, receipt = sample_uk_national_frame(frame, fraction=1.0, seed=42) - assert sampled is frame - assert "normalization_factor" not in receipt - assert receipt["uk_policy"]["spi_replacement_quota_checked"] is True - - def test_spine_source_units_use_raw_family_regions() -> None: frame = _spine_family_frame() units, strata = uk_spine_source_family_units(frame) @@ -490,6 +216,25 @@ def test_spine_sampling_fractional_source_fails_closed() -> None: uk_spine_source_family_units(_spine_family_frame(fractional_source=True)) +def test_spine_sampling_refuses_numeric_region_codes() -> None: + """Region strata labels are formatted values, so an integer code in one + build and a float code in another would silently split a stratum; the + sampler accepts string regions only.""" + + frame = _spine_family_frame() + household = frame.table("household").copy() + household["region"] = np.arange(len(household)) % 3 + numeric = uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=household, + time_period="2023", + household_weights=frame.weights_for("household").values, + ) + with pytest.raises(ValueError, match="region must contain non-empty strings"): + sample_uk_spine_frame(numeric, fraction=0.5, seed=1) + + def test_spine_sampling_orphan_derivative_fails_closed() -> None: # The refusals count families and name no source household (microcosm#931). with pytest.raises(ValueError) as caught: From ebd0f1a208dfdbbdabd65e610b7e3a12055f4c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:20:54 +0000 Subject: [PATCH 022/108] Bind every value-computing helper into the keys of the stages that call it (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A spine stage's key hashes its stage module and the helper modules listed for it, and source_hash does not follow imports, so a stage that called a function in an unlisted module replayed a stale output when that module changed. As María ruled on 8 Nov, this sweep binds those helpers, with a test that derives them. It only changes cache keys. - test_uk_graph gains the derived rule. From the transform graph_kernels builds for each stage, it follows every definition the code names (annotations included, through re-exports and module attributes) into the module defining it, within microcosm.build, to a fixed point, and requires the stage's hash to cover each module it meets. A check that returns and writes nothing, a function returning one column name, a record class of undefaulted fields, a constant of strings and the stage declaration need no binding; a numeric constant does. The other workspace packages stay outside the rule, and the docstring says why. A second test applies the rule to the seven dense kernels that share _PopulationKernel's hash, with the spine gate admission check listed as refusal-only, and a control test pins each form the rule follows and each kind it skips. - graph_kernels: national_frame joins the modules every stage hashes, and the helper table gains 219 bindings of 49 modules in 48 stages, with two shared groups (the vendored-fact reader with the chronicle feed and its schema epochs; frs_spine's pinned tab reader with trace). The six modules the ruling named are bound where stages 26, 42, 43 and 46 compute through them: property_localisation (26), spi_support (26, 42, 43, 46), cgt_imputation (42, 46), cgt_asset_type (46), advani_summers (43) and cgt_structure (42; stages 26 and 46 reach only its closed-world operations check). Stage 14 binds employer_pension_contributions. - The older import-level tests then ask for more: stochastic_assignment in 8 more stages (the support channel's exemption goes), the copy-id table in stages 14 and 18, 15 stages on the copy-id name-only list and four more counting stages. Their re-key tests get new controls and counts. - graph_population: _PopulationKernel hashes rowwise_geography, and the dense test also requires frame_sampling, national_frame, local_authority_input and gates. - Changelog: a new helper-bindings entry; the copy-id entry drops its sentences about the helpers that stayed unbound. What moves: node keys only. Every spine kernel hashes graph_kernels.py, so every spine node re-keys, and with it every node downstream. Every graph_population kernel re-keys (uk.full.sample, normalize, expand, locations, geography_mapping, identity, property.value, property.band, geography.local_authority, geography.brma, geography.rfdr, private_rent, pool and geography_gate), and so do the calibration and size kernels that hash the module (uk.full.dense, selected, calibrated, size_search, size_draw, size_refit and size_checkpoint_import). No value, declaration or generated file moves: uk_spine.json and the H2 fixture hold stage contract hashes, not code hashes, and the fixture check regenerates byte for byte; the source_stages projection and the coverage manifest are current. Tests: test_uk_graph (33 tests) and 15 related engine-free files pass, and ruff is clean. Mutations under scratch: a value helper newly imported into a stage (by name, by module attribute or through a re-export), a numeric constant newly read and a dropped binding (a helper, a group member, national_frame) each fail the stage test, while a helper the transform never names, a check and a constant of strings pass. Dropping rowwise_geography or frame_sampling from the dense hash, or a dense kernel newly naming an unhashed helper, fails the dense test. The H2 fixture check and the H2 parity test pass through the engine slot. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-copy-id-table.changed.md | 2 +- changelog.d/931-uk-helper-bindings.changed.md | 5 + .../build/uk_runtime/graph_kernels.py | 469 ++++++++++-- .../build/uk_runtime/graph_population.py | 14 + .../tests/engine_free/uk/test_uk_graph.py | 671 +++++++++++++++++- 5 files changed, 1072 insertions(+), 89 deletions(-) create mode 100644 changelog.d/931-uk-helper-bindings.changed.md diff --git a/changelog.d/931-uk-copy-id-table.changed.md b/changelog.d/931-uk-copy-id-table.changed.md index 2fa3d56fff..11fef28435 100644 --- a/changelog.d/931-uk-copy-id-table.changed.md +++ b/changelog.d/931-uk-copy-id-table.changed.md @@ -3,4 +3,4 @@ The UK spine mints every copy row's ids from fixed decimal fields declared in `m - **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 on any frame; the row-by-row walk it replaces read a row holding only integer and float columns as float64, inexact above 2^53. The UK modules' refusals on the copy path print counts, never ids: the new ones, and the older checks that listed up to five ids or printed the largest (uniqueness, links, sources, carriers and weights in the clone helpers, the SPI support and spine stages, the row-wise dataset and its tool's dry-run plans, the CGT stages and the spine sampler); the shared graph executor's own EXPAND checks, which every copy stage also passes through, are unchanged, and some of them still name ids. `create_uk_spi_support_tables` no longer takes an `id_multiplier`. The identity tool (`tools/verify_uk_identity_stability.py`) decodes the CGT support copies and the residential arms at their fields' declared radices and records the declared radix as each receipt's `id_multiplier`; it refuses, by count, a spine whose copy ids do not follow the table, as every pre-table spine with copy rows does, and such a spine is receipted with the tool at the commit that built it. - **What moves.** Base FRS rows keep their ids. Every copy row's five id columns now sit at its parent row's ids plus its layer's offset: SPI copies 10^11, band donors 2×10^11, high-value copy k k×10^12, CGT copy k k×10^13, twins 10^17 and arm j j×10^18. Each used to be a power of ten above the largest id its layer saw. Relative id order is unchanged at K=1, so sorts, ties, ranks and folds are unchanged. Values move on copy rows through every draw keyed on the copy's own id: the SPI income stage's second stage, stages 19 to 24 and 27 to 41, every twin's Advani-Summers prior (so the CGT gains layer is re-realised), stages 48 to 54 and policyengine-uk's Warm Home Discount draw on copy households in spine engine calls. Base rows move second-order through pooled steps: the high-value selection, the regional value ranks, the landlord walk, the residential solve, the anchor, the CGT walks and calibration. Draws keyed on `person_source_id` do not move. National ids pass 2^53 from the incidence twin on, and at the CGT support split for any copy at ordinal 901 or more. The released file's identifiers, ranks of the build ids, do not move at K=1, while its `*_build_id` columns carry the new copy ids, so joining copy rows across versions goes through lineage (source id and copy path), not build ids. - **Dense line.** The dense expand keeps its multiplier above the spine's largest id. A spine carrying residential arms (multiples of 10^18), as a production national spine does, leaves no int64 room for a second dense clone, so every dense entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) refuses wherever the dense offset would overflow int64, in one message naming #931 and the clone-first spine. Dense builds with K > 1 of such a spine wait for #931's clone-first spine; K = 1, and a spine whose ids leave room (one without copy rows, say), expand as before. The dataset-size evaluation decodes dense ids back to spine ids on exact integers, 10^19 included. A caller can pass the multiplier the expansion recorded (`weight_tables(id_multiplier=)`); `tools/evaluate_uk_dataset_size.py` infers it from the spine's household ids or the clone-0 household ids, and a dataset the inferred multiplier does not fit is refused by count. A dense file that keeps its true sources decodes its spine stretch again instead of failing. -- **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes. The SPI support and band-donor stages also bind `spi_support`, whose clone helper mints their copies; the high-value split binds `cgt_support`, whose exact-total correction weighs its copies; and the CGT clone and anchor stages, which share `cgt_structure`, bind `rowwise_geography`, whose clone helper mints the twins. The exact-total weighting in `spi_support`, which the twin and the high-value split reach, stays unbound until a wider binding sweep. Every stage whose module or bound helpers import `microcosm.build.stochastic_assignment`, or import another module's function that draws through it, now binds it, so an edit to the keyed draws re-keys each stage they move; the SPI support channel, which shares its module with stage 14's employer pension draw but never calls it, is exempt, and the modules in between (`employer_pension_contributions`, say) are not bound. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. +- **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes. The SPI support and band-donor stages also bind `spi_support`, whose clone helper mints their copies; the high-value split binds `cgt_support`, whose exact-total correction weighs its copies; and the CGT clone and anchor stages, which share `cgt_structure`, bind `rowwise_geography`, whose clone helper mints the twins. Every stage whose module or bound helpers import `microcosm.build.stochastic_assignment`, or import another module's function that draws through it, now binds it, so an edit to the keyed draws re-keys each stage they move. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md new file mode 100644 index 0000000000..a3f218020c --- /dev/null +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -0,0 +1,5 @@ +Every UK spine stage now hashes each module its transform computes through, and the dense population nodes each module theirs do (#931). `source_hash` hashes only the files it is given, so a stage that called a function in a module its hash did not name replayed a stale cached output when that module changed. + +- **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage, follows every definition the code names (annotations included, through re-exports and module attributes) into the module that defines it, within `microcosm.build`, to a fixed point, and checks that the stage's hash covers every module it meets. Five kinds of definition need no binding and are not followed: a check that returns nothing and writes nothing, a function returning one column name, a record class of undefaulted fields, a constant made only of strings, and the stage's own declaration (`country_spec`, `source_manifest` and the spec engine, whose content the node key already hashes as the stage contract). A numeric constant needs its module bound. The workspace's other packages stay outside the rule: `microcosm.graph` and `microcosm.frame` run every node, and `microcosm.fit`'s fitters are bound by hand. A second test applies the rule to the seven dense population kernels that share `_PopulationKernel`'s hash, where the spine gate admission check is listed as refusal-only, and a control test pins each way of naming a helper that the rule follows and each kind it skips. +- **What is bound.** `national_frame` joins the modules every spine stage hashes. The stage helper table gains 219 bindings of 49 modules in 48 stages, among them `property_localisation`, `cgt_structure`, `spi_support`'s exact-total helpers, `cgt_imputation`, `cgt_asset_type` and `advani_summers` in the CGT and high-value stages, and `employer_pension_contributions` in `frs_hmrc_spine_leaves`. Two groups recur: the vendored-fact reader with the chronicle feed and its schema epochs (20 stages), and `frs_spine`'s pinned tab reader with `trace` (14 stages). Because the new helpers import them, `stochastic_assignment` is bound into 8 more stages and the SPI support channel's exemption goes, the copy-id table is bound into `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`, 15 stages join the copy-id test's name-only list, and `frs_spine`, `spi_support_channel`, `spi_income_band_donors` and `uc_deduction_attributes` join the counting stages. The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input` and `gates`. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 21f0174005..992f4db5b7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -51,33 +51,70 @@ ) from microcosm.graph.population import dtype_for_token -from .. import source_lineage, stage_evidence, stochastic_assignment +from .. import ( + chronicle_epoch, + gates, + ledger_targets, + raking, + source_lineage, + stage_evidence, + stochastic_assignment, + target_materialization, + trace, +) from . import ( + advani_summers, + birth_month, + bus_fare_pricing, + bus_support_pricing, bus_use_incidence, + cgt_asset_type, cgt_imputation, + cgt_structure, cgt_support, + chronicle_feed, copy_id_table, + council_tax_constants, + dfe_spring_term, + donor_uprating, + employer_pension_contributions, energy_pricing, + frs_brma, frs_disability, frs_education, frs_hmrc_source, + frs_release, frs_spine, frs_take_up, + gambling_participation, + hmrc_capital_gains, hmrc_company_car_statistics, hmrc_income, hmrc_isa_statistics, hmrc_property_rental, + hmrc_replay, hmrc_uprating, + household_income, household_lineage, ledger_fact_vendoring, legacy_closure_take_up, + national_frame, + nts_car_attributes, occupation_constants, + pension_coherence, property_localisation, + qrf_gates, receipt_counts, + release_input_coverage, rowwise_geography, + spi_band_donors, spi_income, spi_rank_transport, spi_support, + stage_health, + support_clip, + take_up_contract, + terminal_gates, uc_capital_coherence, uc_deduction_attributes, uc_relationships, @@ -170,51 +207,239 @@ uk_engine_population_declaration, ) -# Imported modules are not traversed by ``source_hash``. Bind relationship -# helpers and the adapter's input-retention checks into every consuming stage. +# Imported modules are not traversed by ``source_hash``, so each stage binds +# every module holding code its transform computes through, beyond its own +# module (microcosm#931). test_uk_graph derives that set: it follows what the +# transform names into the module defining it, to a fixed point, and needs no +# binding for a check, a column name, a record or a constant of strings. # Every stage whose module or bound helpers import the shared keyed-draw # helpers (``microcosm.build.stochastic_assignment``), or import another -# module's function that draws through them, binds them, so an edit to how a -# uniform is keyed or a binary is placed re-keys every stage it moves -# (microcosm#931). test_uk_graph derives the set from the imports and names, -# with its reason, each stage that imports a drawing function it never calls. -# The intermediate modules themselves (employer_pension_contributions, say) -# are not bound here. +# module's function that draws through them, binds them as well, so an edit +# to how a uniform is keyed or a binary is placed re-keys every stage it +# moves; test_uk_graph derives that set from the imports and names. + +# A vendored ledger fact is read through ledger_fact_vendoring, which checks +# it against the pinned chronicle feed and the feed's accepted schema epochs. +_FEED_FACT_MODULES = (ledger_fact_vendoring, chronicle_feed, chronicle_epoch) +# A pinned source tab is read through frs_spine's tab reader, which checks the +# tab's digest through trace. +_PINNED_TAB_MODULES = (frs_spine, trace) + _STAGE_HELPER_MODULES = { # The three FRS draw stages' receipts measure their take-up flags through # frs_take_up's signal measurement and count support by the shared - # disclosure rules (microcosm#1095). - "frs_take_up": (receipt_counts, stochastic_assignment), - "frs_person_draws": (frs_take_up, receipt_counts, stochastic_assignment), - "frs_household_draws": (frs_take_up, receipt_counts, stochastic_assignment), + # disclosure rules (microcosm#1095). The flags draw on the take-up + # contract's rates and age children by the DfE's spring-term rule + # (microcosm#931). + "frs_take_up": ( + dfe_spring_term, + receipt_counts, + take_up_contract, + stochastic_assignment, + ), + "frs_person_draws": ( + dfe_spring_term, + frs_take_up, + receipt_counts, + take_up_contract, + stochastic_assignment, + ), + "frs_household_draws": ( + dfe_spring_term, + frs_take_up, + receipt_counts, + take_up_contract, + stochastic_assignment, + ), + # The FRS stages that read raw tabs and normalise their ids through + # frs_spine; the employment stage maps occupations to the shared codes, and + # the disability flags resolve the benefit year's rules from the FRS + # release and annualise on frs_spine's week count (microcosm#931). + "frs_relationships": (*_PINNED_TAB_MODULES, stochastic_assignment), + "frs_employment": ( + *_PINNED_TAB_MODULES, + occupation_constants, + stochastic_assignment, + ), + "frs_council_tax": (*_PINNED_TAB_MODULES, stochastic_assignment), + "frs_disability": (frs_release, frs_spine, stochastic_assignment), + "frs_education": (*_PINNED_TAB_MODULES, stochastic_assignment), # The leaves' employer pension contributions are drawn on keyed uniforms - # through employer_pension_contributions (microcosm#931). - "frs_hmrc_spine_leaves": (frs_hmrc_source, stochastic_assignment), + # through employer_pension_contributions, from the ASHE rates it reads as + # vendored facts; the weekly leaves annualise on frs_spine's week count + # (microcosm#931). Their module, spi_spine, also mints the support + # channel's copies, so the leaves and the SPI income stage bind the copy-id + # table as the support channel does. + "frs_hmrc_spine_leaves": ( + frs_hmrc_source, + copy_id_table, + employer_pension_contributions, + *_FEED_FACT_MODULES, + frs_spine, + stochastic_assignment, + ), # The annual income descriptors' receipt follows the shared disclosure rules - # (microcosm#1164). - "frs_annual_income_spells": (receipt_counts, stochastic_assignment), + # (microcosm#1164); the spells read their tabs and interview dates through + # frs_spine (microcosm#931). + "frs_annual_income_spells": ( + receipt_counts, + *_PINNED_TAB_MODULES, + stochastic_assignment, + ), # The SPI income stage's imputation, band-donor resample and receipt live # in spi_income, so an edit there moves the stage's hash (microcosm#1278), - # and its twin transport in spi_rank_transport (microcosm#840). - "hmrc_spi_income_spine": (spi_income, spi_rank_transport, stochastic_assignment), + # and its twin transport in spi_rank_transport (microcosm#840). The + # imputation refreshes the disability flags, coheres the pension + # contributions on the ASHE rate bands, uprates savings income, floors pay + # on frs_spine's week count and reads its age model and band targets as + # vendored facts and HMRC tables; the stage's parameters bound the band + # donors and the coverage floor, and its result carries the fits' weight + # kinds, gate outcomes and the HMRC replay report (microcosm#931). It + # binds the copy-id table as the leaves do. + "hmrc_spi_income_spine": ( + spi_income, + spi_rank_transport, + copy_id_table, + employer_pension_contributions, + *_FEED_FACT_MODULES, + frs_disability, + frs_release, + frs_spine, + frs_take_up, + gates, + hmrc_income, + hmrc_replay, + hmrc_uprating, + pension_coherence, + qrf_gates, + release_input_coverage, + spi_band_donors, + spi_support, + target_materialization, + stochastic_assignment, + ), # The spine's raw ids sit below the copy-id table's first field, and every # copy layer mints its ids in its own field of the table; the two SPI # support stages mint through the support channel's clone helper, and the # high-value split weighs its copies through the CGT split's exact-total # correction (microcosm#931). The executor reads each id-offset stage's - # declared radix from the table too. - "frs_spine": (uc_relationships, copy_id_table, stochastic_assignment), - "spi_support_channel": (spi_support, copy_id_table), - "spi_income_band_donors": (spi_support, copy_id_table, stochastic_assignment), - "high_value_home_support": (cgt_support, copy_id_table), - "cgt_support_split": (copy_id_table,), - "cgt_incidence_clone": (rowwise_geography, copy_id_table, stochastic_assignment), - "cgt_incidence_anchor": (rowwise_geography, copy_id_table, stochastic_assignment), - "cgt_residential_split": (copy_id_table,), - "frs_legacy_proxies": (*_ENGINE_MODULES,), - "frs_education_grant_split": (*_ENGINE_MODULES,), + # declared radix from the table too. The root reads its tabs through + # trace's digest, ages people by birth month, and caps pension + # contributions with their receipt (microcosm#931). + "frs_spine": ( + uc_relationships, + copy_id_table, + birth_month, + pension_coherence, + receipt_counts, + trace, + stochastic_assignment, + ), + # The support channel reads the take-up population's ages and gates its + # zero-weight strata (microcosm#931). + "spi_support_channel": ( + spi_support, + copy_id_table, + frs_take_up, + gates, + terminal_gates, + stochastic_assignment, + ), + # The band donors read HMRC's ITL band taxpayers as vendored facts and the + # donors' age model through spi_income (microcosm#931). + "spi_income_band_donors": ( + spi_support, + copy_id_table, + *_FEED_FACT_MODULES, + frs_take_up, + spi_income, + stochastic_assignment, + ), + # The exact-total correction the CGT split hands the high-value split is + # spi_support's, and the support stratum is property_localisation's + # (microcosm#931). + "high_value_home_support": ( + cgt_support, + copy_id_table, + property_localisation, + spi_support, + ), + # The CGT stages read HMRC's CGT tables as vendored facts and the + # Advani-Summers tail from advani_summers, take the CGT policy parameters + # and income proxy from cgt_imputation (whose loss relief evaluates a named + # term), and weigh their copies through spi_support's exact-total + # correction; the support split also takes the adult age and oldest-adult + # rule from cgt_structure, and the residential split its logistic from + # cgt_asset_type (microcosm#931). + "cgt_support_split": ( + copy_id_table, + *_FEED_FACT_MODULES, + cgt_imputation, + cgt_structure, + hmrc_capital_gains, + hmrc_income, + spi_support, + target_materialization, + stochastic_assignment, + ), + "cgt_incidence_clone": ( + rowwise_geography, + copy_id_table, + advani_summers, + spi_support, + stochastic_assignment, + ), + "cgt_incidence_anchor": ( + rowwise_geography, + copy_id_table, + advani_summers, + cgt_imputation, + hmrc_capital_gains, + stochastic_assignment, + ), + "cgt_residential_split": ( + copy_id_table, + *_FEED_FACT_MODULES, + cgt_asset_type, + cgt_imputation, + hmrc_capital_gains, + spi_support, + ), + "hmrc_cgt_gains_spine": ( + *_FEED_FACT_MODULES, + advani_summers, + hmrc_capital_gains, + hmrc_income, + target_materialization, + ), + "hmrc_cgt_asset_type_spine": ( + *_FEED_FACT_MODULES, + cgt_imputation, + hmrc_capital_gains, + ), + # The legacy proxies read raw tabs, and the grant split resolves its + # year's rules from the FRS release (microcosm#931). + "frs_legacy_proxies": ( + *_ENGINE_MODULES, + *_PINNED_TAB_MODULES, + stochastic_assignment, + ), + "frs_education_grant_split": (*_ENGINE_MODULES, frs_release), "frs_brma": (*_ENGINE_MODULES, stochastic_assignment), - "was_wealth": (*_ENGINE_MODULES, stochastic_assignment), + # The donor stages read their donor tabs through frs_spine, record each + # fit's weight kind, gate the fits and clip donor support to the donor's + # range (microcosm#931); the WAS recipients' predictors map benefit units to + # households and count family roles. + "was_wealth": ( + *_ENGINE_MODULES, + *_PINNED_TAB_MODULES, + frs_brma, + gates, + qrf_gates, + support_clip, + uc_relationships, + stochastic_assignment, + ), # The LISA stage reads its household donor through the WAS cleaning and its # household predictors through the WAS recipient surface; the anchor counts # HMRC's payers through the typed ISA facts (microcosm#1415). @@ -222,15 +447,28 @@ *_ENGINE_MODULES, was_wealth, hmrc_isa_statistics, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + frs_brma, + gates, + support_clip, + uc_relationships, stochastic_assignment, ), # Landlords' receipts read the vendored PRIS rows through their typed # loader, the tape through the income stage's preparation and uprating, - # and the support channel's names (microcosm#1106). + # and the support channel's names (microcosm#1106); the loader checks the + # feed, the preparation reads the donors' age model over the take-up + # population and uprates savings income, and the fits record their weight + # kinds (microcosm#931). "property_components": ( hmrc_property_rental, spi_income, spi_support, + *_FEED_FACT_MODULES, + frs_take_up, + gates, + hmrc_uprating, stochastic_assignment, ), # The regional values rank as the dense node does and read its knots @@ -239,31 +477,101 @@ # The purchases read the vendored HMRC Stamp Tax rows through the feed-pinned # reader and mark SDLT's regions from the FRS region map (microcosm#1095). "property_purchases": ( - ledger_fact_vendoring, + *_FEED_FACT_MODULES, rowwise_geography, stochastic_assignment, ), - "nts_bus_travel": (*_ENGINE_MODULES, bus_use_incidence, stochastic_assignment), + # The bus travel donors also read published car availability as vendored + # facts, and the recipients' HBAI gross income (microcosm#931). + "nts_bus_travel": ( + *_ENGINE_MODULES, + bus_use_incidence, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + gates, + household_income, + qrf_gates, + support_clip, + stochastic_assignment, + ), # The energy level and prices are energy_pricing's (microcosm#1418): a # code-only change to them must re-key the stage (the independent review - # of microcosm#1420, finding 5). - "lcfs_consumption": (*_ENGINE_MODULES, energy_pricing, stochastic_assignment), - "etb_vat": (*_ENGINE_MODULES, stochastic_assignment), - "etb_services": (*_ENGINE_MODULES, stochastic_assignment), - "uc_reporter_redraw": (uc_relationships, *_ENGINE_MODULES, stochastic_assignment), + # of microcosm#1420, finding 5). The LCFS and ETB donors are uprated + # through donor_uprating on vendored indices, the recipients' household + # income and family roles are predictors, and bus fares are priced over + # calendar and fiscal-year windows; the LCFS stage also rakes its energy, + # assigns car attributes and tops up gambling (microcosm#931). + "lcfs_consumption": ( + *_ENGINE_MODULES, + energy_pricing, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + bus_fare_pricing, + donor_uprating, + gambling_participation, + gates, + household_income, + ledger_targets, + nts_car_attributes, + qrf_gates, + raking, + support_clip, + stochastic_assignment, + ), + "etb_vat": ( + *_ENGINE_MODULES, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + donor_uprating, + gates, + household_income, + support_clip, + uc_relationships, + stochastic_assignment, + ), + "etb_services": ( + *_ENGINE_MODULES, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + bus_fare_pricing, + bus_support_pricing, + donor_uprating, + energy_pricing, + gates, + ledger_targets, + qrf_gates, + support_clip, + uc_relationships, + stochastic_assignment, + ), + # The reporter redraw gates its fits and maps household weights to benefit + # units as the UC capital stage does (microcosm#931). + "uc_reporter_redraw": ( + uc_relationships, + *_ENGINE_MODULES, + qrf_gates, + uc_capital_coherence, + stochastic_assignment, + ), # The SPI benefit pass restores from the FRS twin through the support # channel's lineage, re-derives the disability flags and the own-right # rule, draws over the take-up population and maps household weights to - # benefit units as the UC capital stage does (microcosm#1095). + # benefit units as the UC capital stage does (microcosm#1095); the flags + # resolve their year's rules and week count, and the draws read the + # take-up contract (microcosm#931). "spi_benefit_coherence": ( spi_support, frs_disability, frs_education, frs_take_up, uc_capital_coherence, + frs_release, + frs_spine, + take_up_contract, stochastic_assignment, ), - "uc_capital_coherence": (uc_relationships, stochastic_assignment), + # Unavailable capital is frs_spine's sentinel (microcosm#931). + "uc_capital_coherence": (uc_relationships, frs_spine, stochastic_assignment), # The Universal Credit redraw reads the award through the adapter, the # One engine materialization of the proxy, the claimant role and carer # status; the frs_spine carrier's codes; the receipt's support counts @@ -271,63 +579,85 @@ "uc_limited_capability": (uk_engine_adapter, frs_spine, receipt_counts), # vendored DWP counts, the closure stage's rule and rates, the deduction # attributes, the take-up population and the family-type roles, and counts - # its pools and receipt on the source lineage (microcosm#1334). + # its pools and receipt on the source lineage (microcosm#1334); the + # attributes map benefit units to households and judge their realisation + # by the shared tolerance, and the year's rules come from the FRS release + # (microcosm#931). "uc_take_up": ( *_ENGINE_MODULES, - ledger_fact_vendoring, + *_FEED_FACT_MODULES, legacy_closure_take_up, uc_deduction_attributes, frs_take_up, uc_relationships, uc_capital_coherence, receipt_counts, + frs_brma, + frs_release, + stage_health, stochastic_assignment, ), # The Move to UC claim draw reuses the UC stage's household-to-benefit-unit # weight mapping and reads the vendored DWP rows (microcosm#1095). "uc_legacy_closure_take_up": ( uc_capital_coherence, - ledger_fact_vendoring, + *_FEED_FACT_MODULES, stochastic_assignment, ), # The Pension Credit redraw reuses the UC stage's household-to-benefit-unit - # weight mapping and the adapter's engine materialization. + # weight mapping and the adapter's engine materialization; it reads the + # take-up contract and the year's rules (microcosm#931). "pension_credit_take_up": ( uc_capital_coherence, *_ENGINE_MODULES, + frs_release, + take_up_contract, stochastic_assignment, ), - # The Child Benefit redraw shares the weight mapping and the adapter. + # The Child Benefit redraw shares the weight mapping and the adapter, and + # reads the vendored Child Benefit statistics (microcosm#931). "child_benefit_take_up": ( uc_capital_coherence, *_ENGINE_MODULES, + *_FEED_FACT_MODULES, stochastic_assignment, ), # The Carer's Allowance draw reads the earnings test through the adapter's # engine materialization and counts its receipt's support on the source # lineage (microcosm#1319), and balances its draws through the shared - # placement helper (microcosm#1418). + # placement helper (microcosm#1418); it reads the take-up contract and the + # year's rules (microcosm#931). "carers_allowance_take_up": ( *_ENGINE_MODULES, receipt_counts, + frs_release, + take_up_contract, stochastic_assignment, ), - # The stored attendance is one engine materialization (microcosm#1095). - "private_school_attendance": (*_ENGINE_MODULES,), - # These four stages draw through the keyed-draw helpers and bind nothing - # else (microcosm#931). - "spi_housing_shell": (stochastic_assignment,), - "uc_deduction_attributes": (stochastic_assignment,), - "salary_sacrifice": (stochastic_assignment,), - "student_loans": (stochastic_assignment,), + # The stored attendance is one engine materialization (microcosm#1095), + # whose year's rules come from the FRS release (microcosm#931). + "private_school_attendance": (*_ENGINE_MODULES, frs_release), + # The housing shell keeps council tax bands within the nation and gates + # its fits; the deduction attributes map benefit units to households and + # judge their realisation by the shared tolerance; the salary sacrifice + # reads its bonus anchor as vendored facts and gates its fits; the + # student loans resolve their year's rules (microcosm#931). + "spi_housing_shell": (council_tax_constants, qrf_gates, stochastic_assignment), + "uc_deduction_attributes": (frs_brma, stage_health, stochastic_assignment), + "salary_sacrifice": (*_FEED_FACT_MODULES, qrf_gates, stochastic_assignment), + "student_loans": (frs_release, stochastic_assignment), # The subscriptions walk the CGT bands on its income proxy (with the # proxy's loss relief), read HMRC's ISA tables through their typed loader - # and round their receipts' support (microcosm#1415). + # and round their receipts' support (microcosm#1415); the loss relief + # evaluates its named term, and the donor tabs are pinned (microcosm#931). "isa_subscriptions": ( cgt_imputation, hmrc_income, hmrc_isa_statistics, receipt_counts, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + target_materialization, stochastic_assignment, ), # The points draws are keyed on identity, read the vendored DWP tables and @@ -341,7 +671,8 @@ # index and contribution cap, the SPI channel's EPB and tape-row flag, and # price the cars twice through the engine adapter; their draws are keyed on # identity, and their receipts' support is rounded - # (PolicyEngine/uk-autumn-budget-dashboard-2026#316). + # (PolicyEngine/uk-autumn-budget-dashboard-2026#316). The loader checks the + # feed, and the donor tab is pinned (microcosm#931). "company_cars": ( frs_spine, hmrc_company_car_statistics, @@ -349,16 +680,23 @@ receipt_counts, spi_income, spi_support, + trace, + *_FEED_FACT_MODULES, stochastic_assignment, *_ENGINE_MODULES, ), # The copies' childcare re-draw draws through frs_take_up's childcare # helper, reads each household's clone path, and measures its flags for - # the stage-time signal gate by the shared disclosure rules (microcosm#1095). + # the stage-time signal gate by the shared disclosure rules (microcosm#1095); + # it maps benefit units to households, ages children by the DfE's rule and + # reads the take-up contract (microcosm#931). "childcare_copy_take_up": ( frs_take_up, household_lineage, receipt_counts, + dfe_spring_term, + frs_brma, + take_up_contract, stochastic_assignment, ), # The employment shell fits, scores and draws through microcosm.fit's @@ -376,12 +714,15 @@ # The sub-major draw fits and scores through microcosm.fit's categorical # classifier and held-out scores, reads the shared occupation codes, and # counts its receipt's support by the shared disclosure rules - # (microcosm#1490). + # (microcosm#1490); it reads its donor tab through trace's digest and + # records its fit's weight kind (microcosm#931). "aps_occupation": ( fit_categorical, fit_held_out, occupation_constants, receipt_counts, + gates, + trace, stochastic_assignment, ), } @@ -395,11 +736,14 @@ _COUNTING_MODULES = (receipt_counts, household_lineage, source_lineage) _COUNTING_STAGES = frozenset( { + "frs_spine", "frs_take_up", "frs_person_draws", "frs_household_draws", "frs_annual_income_spells", "hmrc_spi_income_spine", + "spi_support_channel", + "spi_income_band_donors", "company_cars", "spi_housing_shell", "high_value_home_support", @@ -419,9 +763,9 @@ "salary_sacrifice", "carers_allowance_take_up", "childcare_copy_take_up", + "uc_deduction_attributes", "isa_subscriptions", "trading_allowance_receipts", - "frs_spine", "frs_council_tax", "frs_disability", "spi_employment_shell", @@ -474,12 +818,15 @@ def _implementation_hash(kernel: object, stage: str, transform: object | None) - # hermetic registries unhashable and, more importantly, would fail to bind # production edits made elsewhere in that stage's module. A transform may # still declare extra behavior-bearing sources through - # ``graph_implementation_dependencies``. + # ``graph_implementation_dependencies``. Every kernel rebuilds its frame + # on national_frame's schema, and the transforms build, validate and read + # the time period of their frames through it (microcosm#931). dependencies = getattr(transform, "graph_implementation_dependencies", None) return source_hash( type(kernel), stage_evidence, graph_evidence, + national_frame, _stage_artifacts, _mass_log_payload, _stage_module(stage), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index fa08b8beb5..1ec7fffc04 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -45,17 +45,20 @@ from microcosm.graph.randomness import keyed_uniform from microcosm.graph.weight_update import weight_update_receipt +from .. import frame_sampling, gates from . import ( atomic_area_support, atomic_household_identity, brma_area_layer, geography_ladder, local_authority_input, + national_frame, national_sampling, private_rent_localisation, property_localisation, rfdr_area_layer, rowwise_dataset, + rowwise_geography, ) from .geography_ladder import ( UK_GEOGRAPHY_LADDER_COLUMNS, @@ -198,12 +201,23 @@ class _PopulationKernel(KernelBase): def implementation_hash(self) -> str: from . import graph_evidence + # The dense population nodes also compute through what these call: + # the sample's household draw and mass normalisation (frame_sampling), + # the clones' minted ids and copied tables (rowwise_geography), the + # national frame they rebuild, the local authority names the geography + # resolves, and the gate's results (microcosm#931); test_uk_graph + # derives the set from what the kernels name. return source_hash( type(self), geography_ladder, national_sampling, rowwise_dataset, graph_evidence, + frame_sampling, + gates, + local_authority_input, + national_frame, + rowwise_geography, ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index fe32e5a350..87f430295f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -7,6 +7,7 @@ from microcosm.graph import graph_schema, orrery_document from microcosm.graph.canonical import canonical_json +from test_support.microcosm_build.first_party_imports import hashed_module_names from test_support.microcosm_build.uk_graph import * @@ -316,14 +317,37 @@ def changed_helper_bytes(path: Path, helper=helper_path, read=original): #: Stages that bind ``spi_support`` or ``frs_spine``, which read the copy-id #: table, for names that never mint, decode or bound a copy id: constants, #: the support channel's column names and source fill, and frs_spine's tab -#: reader and self-employment rule. They need not bind the table. +#: reader, id normaliser, interview dates and self-employment rule. They need +#: not bind the table; the helper rule below holds that none of them reaches +#: the table's code another way. _COPY_ID_TABLE_NAME_ONLY_STAGES = frozenset( - {"company_cars", "property_components", "spi_benefit_coherence"} + { + "company_cars", + "etb_services", + "etb_vat", + "frs_annual_income_spells", + "frs_council_tax", + "frs_disability", + "frs_education", + "frs_employment", + "frs_legacy_proxies", + "frs_relationships", + "isa_subscriptions", + "lcfs_consumption", + "nts_bus_travel", + "property_components", + "spi_benefit_coherence", + "uc_capital_coherence", + "was_lisa", + "was_wealth", + } ) #: The functions those stages may import from the two helpers. _COPY_ID_TABLE_FREE_HELPER_FUNCTIONS = frozenset( { "fill_support_channel_from_source", + "frs_interview_dates", + "normalize_ids", "read_pinned_tab", "support_channel_column", "support_source_id_column", @@ -402,7 +426,7 @@ def test_copy_id_table_changes_invalidate_every_stage_that_reads_it(monkeypatch) ] controls = [ graph.node(stage).kernel - for stage in ("frs_council_tax", "hmrc_spi_income_spine", "isa_subscriptions") + for stage in ("frs_council_tax", "hmrc_cgt_gains_spine", "isa_subscriptions") ] before = {ref: registry.implementation_hash(ref) for ref in affected + controls} helper_path = Path(copy_id_table.__file__).resolve() @@ -415,7 +439,7 @@ def changed_helper_bytes(path: Path) -> bytes: return content monkeypatch.setattr(Path, "read_bytes", changed_helper_bytes) - assert len(affected) == 8 + assert len(affected) == 10 for ref in affected: assert registry.implementation_hash(ref) != before[ref], ref for ref in controls: @@ -426,22 +450,10 @@ def changed_helper_bytes(path: Path) -> bytes: #: Stages that import a function drawing through the keyed-draw helpers #: (``microcosm.build.stochastic_assignment``) but never call it, so need not #: bind them: the stage's transform, the drawing names its module imports, and -#: why the transform never calls them. -_STOCHASTIC_ASSIGNMENT_UNBOUND: dict[str, tuple[str, frozenset[str], str]] = { - "spi_support_channel": ( - "UKSPISupportChannelStageTransform", - frozenset( - { - "employer_pension_contributions.draw_employer_pension_contributions", - "spi_income.impute_uk_spi_income_support", - } - ), - "its module, spi_spine, also holds frs_hmrc_spine_leaves' employer " - "pension draw and hmrc_spi_income_spine's SPI income imputation, both " - "of which bind the helpers; the support channel's transform calls " - "neither", - ), -} +#: why the transform never calls them. None at present: the support channel, +#: the one stage listed before, binds frs_take_up, which imports the helpers, +#: and so binds them too (microcosm#931). +_STOCHASTIC_ASSIGNMENT_UNBOUND: dict[str, tuple[str, frozenset[str], str]] = {} class _DrawFacts(NamedTuple): @@ -625,10 +637,13 @@ def test_every_stage_that_draws_through_stochastic_assignment_binds_it() -> None if stochastic_assignment in helpers } assert binding == set(reach) - set(_STOCHASTIC_ASSIGNMENT_UNBOUND) - # The employer pension draw reaches stage 14 through spi_spine alone. - assert reach["frs_hmrc_spine_leaves"] >= { - "employer_pension_contributions.draw_employer_pension_contributions" - } + # Stage 14's module imports the employer pension draw, which draws through + # the helpers; the stage binds the draw's module as well. + spi_spine = graph_kernels._stage_module("frs_hmrc_spine_leaves").__name__ + employer = "microcosm.build.uk_runtime.employer_pension_contributions" + drawing = _drawing_names([spi_spine])[employer] + assert "draw_employer_pension_contributions" in drawing + assert "frs_hmrc_spine_leaves" in reach # An exempt stage imports exactly the drawing names its reason covers, and # its transform names none of them, directly or through its module. for stage, (transform, names, _) in _STOCHASTIC_ASSIGNMENT_UNBOUND.items(): @@ -651,7 +666,7 @@ def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( for stage, helpers in graph_kernels._STAGE_HELPER_MODULES.items() if stochastic_assignment in helpers ) - # Stage 14 draws through the employer pension helper, which binds nothing. + # Stage 14 draws through the employer pension helper, which it binds too. assert "frs_hmrc_spine_leaves" in stages # The root stage's node is the CREATE kernel. affected = [ @@ -660,8 +675,8 @@ def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( ] # Three stages that draw through nothing of it. control_stages = ( - "frs_council_tax", - "cgt_support_split", + "regional_home_values", + "hmrc_cgt_gains_spine", "hmrc_cgt_asset_type_spine", ) drawing = _stages_drawing_through_stochastic_assignment() @@ -859,6 +874,608 @@ def changed_module_bytes(path): assert registry.implementation_hash(ref) == before[ref], ref +# The rule behind the bindings above, for every helper (microcosm#931): +# ``source_hash`` does not follow imports, so a node's hash must name every +# module whose code can change what the node computes. The tests below +# derive those modules from what a stage's transform, or a dense population +# kernel, names, and follow each named definition into the module that +# defines it, through re-exports and module attributes, to a fixed point. + +#: The package the rule follows: microcosm.build holds every UK stage and its +#: helpers. The workspace's other packages stay outside it. microcosm.graph +#: and microcosm.frame are the executor and the Frame that every node runs +#: on, so binding them would re-key every node of every graph on any edit to +#: them; microcosm.fit's fitters, shared by every country, are bound by hand +#: (the employment shells above) and not derived here. +_HELPER_SCOPE = "microcosm.build." +#: Where the rule stops. What a stage reads from these is its own +#: declaration, which its node key already hashes as the stage contract: its +#: operations, artifacts and resource pins (graph._stage_contract_sha256). +_DECLARATION_MODULES = ( + "microcosm.build.country_spec", + "microcosm.build.source_manifest", + "microcosm.build.spec_engine", +) +#: Checks that read values only to decide whether to refuse, each with why it +#: cannot change what its caller computes. The rule does not follow them; the +#: test checks that each returns nothing and writes nothing. +_REFUSAL_ONLY_HELPERS: dict[tuple[str, str], str] = { + ( + "microcosm.build.uk_runtime.graph_evidence", + "require_uk_spine_gate_admission", + ): ( + "refuses a node unless the spine gates admitted its input; the dense " + "sample node calls it before it computes anything" + ), +} +#: Method names that change their receiver in place. +_IN_PLACE_CALLS = frozenset( + { + "__setattr__", + "__setitem__", + "add", + "append", + "clear", + "discard", + "extend", + "fill", + "insert", + "pop", + "popitem", + "remove", + "setdefault", + "sort", + "update", + } +) + + +class _ModuleDefinitions(NamedTuple): + """What one module imports, anywhere in it, and defines at its top level.""" + + imports: dict[str, tuple[str, str]] + modules: dict[str, str] + stars: tuple[str, ...] + definitions: dict[str, object] + + +@functools.cache +def _is_module(name: str) -> bool: + """Whether a first-party dotted name is a module. The rule never follows + another package, so it never imports one to find out.""" + + import importlib.util + + if not name.startswith("microcosm."): + return False + parent = name.rpartition(".")[0] + try: + if parent: + spec = importlib.util.find_spec(parent) + if spec is None or spec.submodule_search_locations is None: + return False + return importlib.util.find_spec(name) is not None + except (ImportError, ValueError): + return False + + +@functools.cache +def _module_definitions(name: str, scope: str) -> _ModuleDefinitions | None: + import ast + import importlib.util + + if not name.startswith(scope): + return None + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): + return None + if spec is None or not (spec.origin or "").endswith(".py"): + return None + package = name if spec.submodule_search_locations else name.rpartition(".")[0] + tree = ast.parse(Path(spec.origin).read_text(encoding="utf-8")) + imports: dict[str, tuple[str, str]] = {} + modules: dict[str, str] = {} + stars: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.level: + base = package.rsplit(".", node.level - 1)[0] + origin = ".".join(part for part in (base, node.module) if part) + else: + origin = node.module or "" + for alias in node.names: + if alias.name == "*": + stars.append(origin) + elif _is_module(f"{origin}.{alias.name}"): + modules[alias.asname or alias.name] = f"{origin}.{alias.name}" + else: + imports[alias.asname or alias.name] = (origin, alias.name) + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.asname: + modules[alias.asname] = alias.name + else: + head = alias.name.partition(".")[0] + modules[head] = head + definitions: dict[str, object] = {} + for node in tree.body: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + definitions[node.name] = node + elif isinstance(node, ast.Assign | ast.AnnAssign): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + for sub in ast.walk(target): + if isinstance(sub, ast.Name): + definitions[sub.id] = node + return _ModuleDefinitions(imports, modules, tuple(stars), definitions) + + +def _resolve(module: str, name: str, scope: str) -> tuple[str, str] | None: + """The top-level definition ``name`` refers to in ``module``, through + re-exports; None for a module or anything outside ``scope``.""" + + pending, seen = [(module, name)], set() + while pending: + item = pending.pop() + if item in seen: + continue + seen.add(item) + module, name = item + facts = _module_definitions(module, scope) + if facts is None: + continue + if name in facts.definitions: + return item + if name in facts.imports: + pending.append(facts.imports[name]) + else: + pending.extend((star, name) for star in facts.stars) + return None + + +def _named(module: str, node, scope: str) -> set[tuple[str, str]]: + """The top-level definitions ``node`` names, annotations included: a bare + name, or a name read by attribute on a module, each resolved through + re-exports.""" + + import ast + + facts = _module_definitions(module, scope) + found = set() + for sub in ast.walk(node): + if isinstance(sub, ast.Name): + hit = _resolve(module, sub.id, scope) + elif isinstance(sub, ast.Attribute): + chain = [] + value = sub + while isinstance(value, ast.Attribute): + chain.insert(0, value.attr) + value = value.value + target = ( + facts.modules.get(value.id) if isinstance(value, ast.Name) else None + ) + while target and chain and _is_module(f"{target}.{chain[0]}"): + target = f"{target}.{chain.pop(0)}" + hit = _resolve(target, chain[0], scope) if target and chain else None + else: + continue + if hit is not None: + found.add(hit) + return found + + +def _writes_nothing(node) -> bool: + """Whether a function returns nothing but None or an argument it never + rebinds, and writes through no subscript, attribute or in-place call.""" + + import ast + + arguments = node.args + parameters = { + argument.arg + for argument in ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + *(a for a in (arguments.vararg, arguments.kwarg) if a is not None), + ) + } + rebound = { + sub.id + for sub in ast.walk(node) + if isinstance(sub, ast.Name) and isinstance(sub.ctx, ast.Store | ast.Del) + } + for sub in ast.walk(node): + if isinstance(sub, ast.Return) and sub.value is not None: + value = sub.value + if not ( + (isinstance(value, ast.Constant) and value.value is None) + or (isinstance(value, ast.Name) and value.id in parameters - rebound) + ): + return False + elif isinstance(sub, ast.Yield | ast.YieldFrom | ast.Global | ast.Nonlocal): + return False + elif isinstance(sub, ast.Assign | ast.AugAssign | ast.AnnAssign | ast.Delete): + targets = ( + sub.targets + if isinstance(sub, ast.Assign | ast.Delete) + else [sub.target] + ) + if any( + isinstance(part, ast.Subscript | ast.Attribute) + for target in targets + for part in ast.walk(target) + ): + return False + elif isinstance(sub, ast.keyword) and sub.arg == "inplace": + return False + elif isinstance(sub, ast.Call): + function = sub.func + if isinstance(function, ast.Attribute) and function.attr in _IN_PLACE_CALLS: + return False + if isinstance(function, ast.Name) and function.id in { + "delattr", + "eval", + "exec", + "setattr", + }: + return False + return True + + +def _string_expression(node) -> bool: + """A string literal, an f-string of names or of one call on names, or a + concatenation of them.""" + + import ast + + if isinstance(node, ast.Constant): + return isinstance(node.value, str) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return _string_expression(node.left) and _string_expression(node.right) + if not isinstance(node, ast.JoinedStr): + return False + for part in node.values: + value = part.value if isinstance(part, ast.FormattedValue) else part + if isinstance(value, ast.Call): + if not isinstance(value.func, ast.Name) or value.keywords: + return False + value = value.args[0] if len(value.args) == 1 else None + if not isinstance(value, ast.Name | ast.Constant): + return False + return True + + +def _only_strings(node) -> bool: + """Whether an expression is built of string literals, None, and names, + in literal collections: a name, a code, or a collection of them.""" + + import ast + + collections = {"MappingProxyType", "dict", "frozenset", "list", "set", "tuple"} + for sub in ast.walk(node): + if isinstance(sub, ast.Constant): + if not (sub.value is None or isinstance(sub.value, str)): + return False + elif isinstance(sub, ast.Call): + if not (isinstance(sub.func, ast.Name) and sub.func.id in collections): + return False + elif not isinstance( + sub, + ast.Name + | ast.Attribute + | ast.Tuple + | ast.List + | ast.Set + | ast.Dict + | ast.JoinedStr + | ast.FormattedValue + | ast.expr_context, + ): + return False + return True + + +_HELPER_KINDS: dict[tuple[str, str, str], str | None] = {} + + +def _helper_kind(module: str, name: str, scope: str) -> str: + """How a top-level definition can bear on a value. + + ``value`` unless it is provably one of: a ``check`` (a function that + writes nothing and names only definitions of these kinds, so it can only + refuse); a ``column name`` (a function returning one string built from + its arguments); a ``record`` (a class of undefaulted fields, no methods); + ``names`` (a constant of strings: a column name, a code, or a collection + of them). A numeric constant is a value. A cycle reads as ``value``. + """ + + key = (module, name, scope) + if key in _HELPER_KINDS: + return _HELPER_KINDS[key] or "value" + _HELPER_KINDS[key] = None + _HELPER_KINDS[key] = _classify_helper(module, name, scope) + return _HELPER_KINDS[key] + + +def _names_only_harmless(module: str, node, scope: str) -> bool: + return all( + found[0].startswith(_DECLARATION_MODULES) + or _helper_kind(*found, scope) != "value" + for found in _named(module, node, scope) + ) + + +def _classify_helper(module: str, name: str, scope: str) -> str: + import ast + + node = _module_definitions(module, scope).definitions[name] + if isinstance(node, ast.Assign | ast.AnnAssign): + if node.value is None or ( + _only_strings(node.value) + and _names_only_harmless(module, node.value, scope) + ): + return "names" + return "value" + if isinstance(node, ast.ClassDef): + fields_only = all( + (isinstance(item, ast.Expr) and isinstance(item.value, ast.Constant)) + or isinstance(item, ast.Pass) + or (isinstance(item, ast.AnnAssign) and item.value is None) + for item in node.body + ) + if fields_only and all( + _names_only_harmless(module, item, scope) + for item in (*node.bases, *node.keywords, *node.decorator_list) + ): + return "record" + return "value" + body = list(node.body) + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + ): + body = body[1:] + if ( + len(body) == 1 + and isinstance(body[0], ast.Return) + and _string_expression(body[0].value) + and _names_only_harmless(module, body[0].value, scope) + ): + return "column name" + if _writes_nothing(node) and _names_only_harmless(module, node, scope): + return "check" + return "value" + + +def _helper_reach( + root: tuple[str, str], scope: str = _HELPER_SCOPE +) -> dict[str, frozenset[str]]: + """Every module holding a definition that can change what ``root`` + computes, with those definitions: ``root`` itself, and each ``value`` + definition it names, and they name, to a fixed point. The declaration + modules and the refusal-only checks are not followed.""" + + reached: dict[str, set[str]] = {} + pending, seen = [root], set() + while pending: + item = pending.pop() + if item in seen: + continue + seen.add(item) + module, name = item + if module.startswith(_DECLARATION_MODULES) or item in _REFUSAL_ONLY_HELPERS: + continue + if item != root and _helper_kind(module, name, scope) != "value": + continue + reached.setdefault(module, set()).add(name) + node = _module_definitions(module, scope).definitions[name] + pending.extend(_named(module, node, scope)) + return {module: frozenset(names) for module, names in reached.items()} + + +def _stage_transforms() -> dict[str, tuple[str, str]]: + """Each spine stage's transform, as graph_kernels builds it for the + hermetic registry and the legacy plan: ``(module, name)``.""" + + import ast + + from microcosm.build.uk_runtime import graph_kernels + + tree = ast.parse(Path(graph_kernels.__file__).read_text(encoding="utf-8")) + package = graph_kernels.__name__.rpartition(".")[0] + transforms = {} + for function in tree.body: + if not isinstance(function, ast.FunctionDef) or function.name not in { + "_fixture_implementations", + "fixture_stage_plan_inputs", + }: + continue + local = { + alias.asname or alias.name: (f"{package}.{sub.module}", alias.name) + for sub in ast.walk(function) + if isinstance(sub, ast.ImportFrom) and sub.level == 1 + for alias in sub.names + } + built = [] + for sub in ast.walk(function): + if isinstance(sub, ast.Dict): + built.extend(zip(sub.keys, sub.values, strict=True)) + elif isinstance(sub, ast.Assign) and isinstance( + sub.targets[0], ast.Subscript + ): + built.append((sub.targets[0].slice, sub.value)) + for key, value in built: + if ( + isinstance(key, ast.Constant) + and isinstance(value, ast.Call) + and isinstance(value.func, ast.Name) + and value.func.id in local + ): + transforms[key.value] = _resolve(*local[value.func.id], _HELPER_SCOPE) + return transforms + + +def test_every_stage_hashes_each_helper_its_transform_computes_through() -> None: + """A stage's hash names every module holding code its transform can run + to compute a value, so an edit to any of them re-keys the stage instead + of replaying a stale output (microcosm#931). + + The rule starts at the transform graph_kernels builds for the stage and + follows every definition it names, annotations included (an injected + contract or table is called through them), into the module defining it. + It needs no binding, and does not follow, a definition that cannot change + a value: a check, a column name, a record, a constant of strings (see + ``_helper_kind``), or a stage declaration. A numeric constant needs its + module bound. The rule starts at the transform, not at the kernel that + runs it: the kernel's code, in graph_kernels, which every stage hashes, + builds every stage's transform for the hermetic registry, so following it + would bind every helper into every stage. + + What the rule cannot see: a call through an object whose class the code + never names, a name looked up by string, and data files read by path. + """ + from microcosm.build.uk_runtime import graph_kernels + + graph = uk_spine_graph() + registry = uk_registry(graph=graph) + transforms = _stage_transforms() + assert set(transforms) == set(graph_kernels._STAGE_MODULES) + unhashed = {} + for stage, transform in sorted(transforms.items()): + assert transform[0] == graph_kernels._stage_module(stage).__name__, stage + # The root stage's node is the CREATE kernel. + node = graph.node("create_uk_frs" if stage == "frs_spine" else stage) + hashed = hashed_module_names(registry.get(node.kernel), graph_kernels) + missing = sorted(set(_helper_reach(transform)) - hashed) + if missing: + unhashed[stage] = missing + assert not unhashed, unhashed + + +def test_the_refusal_only_helpers_return_and_write_nothing() -> None: + for module, name in _REFUSAL_ONLY_HELPERS: + node = _module_definitions(module, _HELPER_SCOPE).definitions[name] + assert _writes_nothing(node), (module, name) + + +def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( + tmp_path, monkeypatch +) -> None: + """Control: a helper called by name, through a re-export, by attribute on + its module, through an alias or a computed constant, or held only in an + annotation is reached, as is a numeric constant; a check, a column name, + a record, a constant of strings, and a helper the root never names are + not.""" + import uuid + + scratch = f"helper_rule_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / scratch + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + sources = { + "direct": "def draw(x):\n return x + 1\n", + "origin": "def total(x):\n return sum(x)\n", + "reexport": "from .origin import total\n", + "attribute": "def share(x):\n return x / 2\n", + "alias": "def _scale(x):\n return x * 3\n\nscale = _scale\n", + "computed": "def _build():\n return {'a': 1}\n\nTABLE = _build()\n", + "collaborator": ( + "class Contract:\n def rate(self, key):\n return 0.5\n" + ), + "numeric": "FLOOR = 10\n", + "harmless": ( + "COLUMN = 'household_weight'\n" + "CODES = frozenset({'A', 'B'})\n\n" + "def check(frame):\n" + " if frame is None:\n" + " raise ValueError('no frame')\n\n" + "def column(entity):\n" + " return f'{entity}_support_channel'\n\n" + "class Record:\n" + " value: float\n" + ), + "unused": "def never(x):\n return x - 1\n", + "stage": ( + "from . import attribute\n" + "from .alias import scale\n" + "from .collaborator import Contract\n" + "from .computed import TABLE\n" + "from .direct import draw\n" + "from .harmless import CODES, COLUMN, Record, check, column\n" + "from .numeric import FLOOR\n" + "from .reexport import total\n" + "from .unused import never\n\n" + "class Transform:\n" + " contract: Contract\n\n" + " def __call__(self, frame):\n" + " check(frame)\n" + " record = Record(value=draw(1) + total([1]))\n" + " rate = self.contract.rate(column('household'))\n" + " return attribute.share(scale(rate)) + TABLE['a'] + FLOOR, (\n" + " record, COLUMN, CODES\n" + " )\n\n" + "def helper():\n" + " return never(1)\n" + ), + } + for module, source in sources.items(): + (package / f"{module}.py").write_text(source) + prefix = f"microcosm.{scratch}." + reach = _helper_reach((f"{prefix}stage", "Transform"), scope=prefix) + assert {module.removeprefix(prefix): names for module, names in reach.items()} == { + "stage": {"Transform"}, + "direct": {"draw"}, + "origin": {"total"}, + "attribute": {"share"}, + "alias": {"scale", "_scale"}, + "computed": {"TABLE", "_build"}, + "collaborator": {"Contract"}, + "numeric": {"FLOOR"}, + } + + +def test_the_dense_population_kernels_hash_each_helper_they_compute_through() -> None: + """The same rule for the dense population kernels that share + ``_PopulationKernel``'s hash (sample, normalize, expand, locations, + geography mapping, pool and gate): every module holding code one of them + can run to compute a value is in the hash they share (microcosm#931). + The rule starts at each kernel's class; the spine gate admission check + the sample kernel calls is refusal-only and is not followed.""" + import inspect + + from microcosm.build.uk_runtime import graph_population, rowwise_geography + + kernels = [ + member + for _, member in inspect.getmembers(graph_population, inspect.isclass) + if member.__module__ == graph_population.__name__ + and issubclass(member, graph_population._PopulationKernel) + and member is not graph_population._PopulationKernel + and member.implementation_hash + is graph_population._PopulationKernel.implementation_hash + ] + assert {kernel.ref for kernel in kernels} >= { + "uk.full.sample@1", + "uk.full.expand@1", + "uk.full.geography_gate@1", + } + unhashed = {} + for kernel in kernels: + hashed = hashed_module_names(kernel.__new__(kernel), graph_population) + # The dense clones are minted by rowwise_geography's helpers. + assert rowwise_geography.__name__ in hashed + missing = sorted( + set(_helper_reach((graph_population.__name__, kernel.__name__))) - hashed + ) + if missing: + unhashed[kernel.ref] = missing + assert not unhashed, unhashed + + def test_uk_graph_json_round_trip_is_canonical() -> None: graph = uk_spine_graph() serialized = graph_to_json(graph) From 5965036f4b10a53f792447a0190035f787cb4822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:25:34 +0000 Subject: [PATCH 023/108] Declare the CGT copy-count snap and hold the K = 1 recount to the tolerance (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was to be the commit that keeps stage 42's exact-total correction off the support copies, as María ruled on 8 Nov: try the other unselected households, then the selected households left whole, never a root or copy. On synthetic frames that rule cannot always make the total exact. In the CGT nesting scenario at K = 7 and 13, every household the split leaves whole sits in the first half of NumPy's pairwise sum. That half lies in the total's binade and the second half carries exactly half an ulp, so round-half-to-even skips the target: no change to the whole households can reach it, and only a copy in the second half can. The rule as written refuses those frames, and 1 to 3 of 24 random 150-household frames at K = 1, 3, 7, 11 and 13. So the correction is unchanged here and goes back to María. This commit lands the two rulings that do not depend on it. - copy_rule names the K-ulp snap ruled on 7 Nov, identically in the code dictionary, sources.yaml and source_stages.json: "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies". The stage notes say "the incidence clone stage (cgt_incidence_clone)" where they meant it. - The identity tool's _e8_support_split holds the stored count between the counts at the folded weight times 1 - 1e-9 and 1 + 1e-9 at K = 1 as well. At K = 1 the fold can land ulps from w: for a weight a few ulps above a multiple of 60 (about half of them in a synthetic sweep), or where the correction fell on a copy. On an exact multiple a moved count is then left to the flag and family-weight checks. The docstring, the receipt's tolerance policy and the test class docstring say so. - The split's docstring and _exact_support_weights' no longer claim the correction always keeps every copy at w / n. - test_at_one_clone_the_count_check_stays_exact becomes test_a_count_moved_on_an_exact_multiple_is_left_to_the_flags, at K = 1 and 3. A new test folds the 900 family just above 900 at K = 1: green, with the exact recount one copy higher. - The changelog fragment describes the snap clause, the K = 1 tolerance and the unchanged correction. What moves: no stage value at any K. The identity tool at K = 1: of 25 synthetic frames as built, the 2 it flagged are green now, both false positives; every tampered frame still fails, but a count moved on an exact multiple is reported by the flags and family weights, not the count check. Its K > 1 verdicts are unchanged. Re-keys: the implementation hashes of cgt_support_split and of high_value_home_support, which binds cgt_support; cgt_support_split's stage contract digest, with its texts; and every node keyed on the UK solve fingerprint, which covers sources.yaml, source_stages.json and the coverage manifest, the spine gates among them, so in a gated build every spine node from stage 14 on re-keys. Regenerated: source_stages.json (copy_rule and notes), the coverage manifest (25 source digests) and the H2 fixture (the two declaration lines in fixture.json and the stage's contract hash in uk_spine.json). Tests: 63 touched engine-free files pass (2,745 tests). The new K = 1 test fails against the parent's tool. The source_stages projection, the coverage manifest and the H2 fixture checks pass, and H2 parity passes in the engine slot. ci_test_plan verify and ruff are clean. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 7 +- .../uk/release_input_coverage_manifest.json | 50 ++++++------- .../src/microcosm/build/uk/source_stages.json | 4 +- .../src/microcosm/build/uk/spec/sources.yaml | 4 +- .../microcosm/build/uk_runtime/cgt_support.py | 19 +++-- .../engine_free/uk/test_uk_cgt_support.py | 4 +- .../uk/test_uk_identity_stability_receipts.py | 65 ++++++++++------ .../parity/uk_spine/sources/fixture.json | 4 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- tools/verify_uk_identity_stability.py | 75 ++++++++++--------- 10 files changed, 135 insertions(+), 99 deletions(-) diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index b7cea3f572..58f7cc7c33 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,5 +1,6 @@ The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. -- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt is unchanged. Its count check stays exact at K = 1. At K > 1 it accepts a family's stored count within the family weights' relative tolerance of 1e-9, because folding the family's copies back re-sums them, which drifts by more ulps than the snap absorbs. -- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies". The stage notes give the cap as 60 divided by the geographic clone count before the incidence clone. -- **What moves.** No value moves at K = 1. On synthetic frames, the stage's output and the identity tool's receipt are unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The stage evidence gains `clone_count`. `cgt_support_split`'s stage contract moves with its texts. Its implementation hash moves, and so does `high_value_home_support`'s, which binds `cgt_support`. The spine gates hash `stage_health` and the country spec, both of which moved, so their keys move, and with them every spine stage after the assembled gate. The coverage manifest's source digests and the H2 fixture's declarations and contract hash are regenerated. +- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. At every K its count check accepts a family's stored count within the family weights' relative tolerance of 1e-9, because folding the family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where the stage's exact-total correction fell on a copy, and at K > 1 by more than the snap absorbs. On an exact multiple of 60 the check therefore accepts n or n + 1 copies, and a count moved there is reported by the flag and family-weight checks instead. The receipt's tolerance policy says so. +- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies". The stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. +- **Exact total.** The exact-total correction is unchanged. It lands on the heaviest unselected household, so every copy weighs bitwise w / n, except where that household cannot carry it: spi_support's shared fallback then places it, and that row can be a root or a copy, a few ulps off w / n. +- **What moves.** No value moves at K = 1. On synthetic frames, the stage's output and the identity tool's receipt are unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. At K = 1 the identity tool no longer reports a count problem for a family that folds back within the tolerance of its stored count. The stage evidence gains `clone_count`. `cgt_support_split`'s stage contract moves with its texts. Its implementation hash moves, and so does `high_value_home_support`'s, which binds `cgt_support`. The spine gates hash `stage_health` and the country spec, both of which moved, so their keys move, and with them every spine stage after the assembled gate. The coverage manifest's source digests and the H2 fixture's declarations and contract hash are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index e7a7914edc..3c84db5f8c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "d43415b009e1da8e26ad9501da7b1d3591a0886ae0546e148beab2aa928f518b", + "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index e8a1b802d4..2b9b8798ef 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -7398,7 +7398,7 @@ ], "selection": "within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "maximum_copy_weight": 60.0, - "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies", "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "flag_column": "household_is_cgt_support_copy", "copies_column": "cgt_support_copies", @@ -7416,7 +7416,7 @@ "cgt_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." }, { "stage": "cgt_incidence_clone", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 9d3f8e831d..88fe7c56ec 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -6011,7 +6011,7 @@ stages: - non_residential_property_value selection: within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused maximum_copy_weight: 60.0 - copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies + copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies id_remapping: copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused flag_column: household_is_cgt_support_copy copies_column: cgt_support_copies @@ -6026,7 +6026,7 @@ stages: - cgt_support_copies - cgt_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' + notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' - stage: cgt_incidence_clone survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index 928716e6e4..b7f2ba9eb6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -584,9 +584,11 @@ def split_cgt_support_households( ``k`` in ``cgt_support_copy_index`` and carries a full copy of the household's person and benefit-unit rows. The exact-total correction lands on the heaviest unselected incumbent so every copy weight stays - bitwise ``w / n``. Refuses a frame the incidence clone stage - (``cgt_incidence_clone``) has already seen or one this stage has already - split. + bitwise ``w / n``, unless that household cannot carry it: then + spi_support's shared fallback places it, and that row can be a root or + copy (:func:`_exact_support_weights`). Refuses a frame the incidence + clone stage (``cgt_incidence_clone``) has already seen or one this stage + has already split. """ validate_uk_national_frame(frame) @@ -766,8 +768,11 @@ def _exact_support_weights( """Importance weights summing bitwise to ``target``. The correction lands on the heaviest unselected incumbent so every split - weight stays exactly ``w / n``; a frame with no unselected positive - incumbent falls back to the shared multi-candidate correction. + weight stays exactly ``w / n``. When that correction is not representable, + or no unselected incumbent is positive, it falls back to spi_support's + shared multi-candidate correction, whose candidates (the heaviest, last, + first, lightest and middle positive rows) can be a root or copy of a + split household. """ if unselected.size: @@ -834,7 +839,9 @@ def cgt_support_split_operation_parameters() -> tuple[ "maximum_copy_weight": CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, "copy_rule": ( "copies = ceil(geographic clone count x household weight / " - "maximum_copy_weight); every copy carries weight / copies" + "maximum_copy_weight), the quotient snapped to the nearest " + "integer within clone-count ulps when the clone count " + "exceeds one; every copy carries weight / copies" ), "id_remapping": ( "copy-id table field cgt_support_split; copy k takes every " diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 59be3c5aba..7c95aad74f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -1577,7 +1577,9 @@ def test_operation_dictionary_restates_the_reviewed_design(): assert payload["maximum_copy_weight"] == 60.0 assert payload["copy_rule"] == ( "copies = ceil(geographic clone count x household weight / " - "maximum_copy_weight); every copy carries weight / copies" + "maximum_copy_weight), the quotient snapped to the nearest integer " + "within clone-count ulps when the clone count exceeds one; every copy " + "carries weight / copies" ) assert payload["flag_column"] == HOUSEHOLD_IS_CGT_SUPPORT_COPY assert payload["copies_column"] == CGT_SUPPORT_COPIES_COLUMN diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 28296881d3..447dedf9de 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1140,7 +1140,7 @@ class TestSupportSplitRecomputeAcrossGeographicClones: The stage reads ``K`` from its input and counts ``ceil(K x w / 60)`` copies (microcosm#931); the tool reads ``K`` from the pre-split table the same way. At one clone its receipt is the one the rule before ``K`` gave. - At ``K > 1`` its count check holds the stored count to the family + At every ``K`` its count check holds the stored count to the family weights' relative tolerance of 1e-9, since the fold re-sums the copies. """ @@ -1256,28 +1256,51 @@ def test_a_count_moved_off_a_non_multiple_is_reported( assert problems["support_split_copies_stored"] == 1, moved - def test_at_one_clone_the_count_check_stays_exact(self) -> None: - # Household 1 (900: 15 copies) moved to 16. At one clone the count - # check reports it, as before the clone count. At three, 3 x w / 60 - # is exactly 15 and the tolerance cannot tell 15 from 16; the flags - # and the family weights report the family instead. - for clone_count, counted in ((1, True), (3, False)): - _, frame = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) - household = frame.table("household").copy() - household.loc[household["household_id"] == 1, CGT_SUPPORT_COPIES_COLUMN] = ( - 16 - ) + @pytest.mark.parametrize("clone_count", [1, 3]) + def test_a_count_moved_on_an_exact_multiple_is_left_to_the_flags( + self, clone_count: int + ) -> None: + # Household 1 (900: 15 copies) moved to 16. K x w / 60 is exactly 15 + # at one clone and at three, and at every clone count the tolerance + # cannot tell 15 from 16 (microcosm#931 review), so the count check + # passes; the flags and the family weights report the family instead. + _, frame = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) + household = frame.table("household").copy() + household.loc[household["household_id"] == 1, CGT_SUPPORT_COPIES_COLUMN] = 16 - _, problems = _exhausting_receipt( - _rebuild(frame, household=household), _NESTING_SCENARIO - ) + _, problems = _exhausting_receipt( + _rebuild(frame, household=household), _NESTING_SCENARIO + ) - assert ("support_split_copies_stored" in problems) is counted - assert problems["support_split_flags"] == { - "copies_disagreeing_with_root": 14, - "families_with_missing_or_extra_copies": 1, - } - assert problems["support_split_family_weights"] == 15 + assert "support_split_copies_stored" not in problems + assert problems["support_split_flags"] == { + "copies_disagreeing_with_root": 14, + "families_with_missing_or_extra_copies": 1, + } + assert problems["support_split_family_weights"] == 15 + + def test_at_one_clone_a_family_folded_just_above_a_multiple_is_green( + self, + ) -> None: + # One copy of household 1 (900: 15 copies of 60) carries a rounding + # amount more, as where the stage's exact-total correction falls on a + # copy, so the family folds to just above 900, where an exact recount + # gives 16. The count check holds the stage's 15 to the tolerance at + # one clone too (microcosm#931 review), so the receipt is green; its + # exact recount still records the extra copy. + split, frame = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + (copy,) = copy_id_table.mint([1], "cgt_support_split", [1]) + weights[ids == copy] *= 1.0 + 1e-12 + + receipt, problems = _exhausting_receipt( + _rebuild(frame, household_weights=weights), _NESTING_SCENARIO + ) + + assert problems == {} + assert receipt["copies"] == split.copies_created == 259 + assert receipt["copies_recomputed"] == 260 # --- copy ids at the copy-id table's declared radices (microcosm#931) ---------- diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index ead642a0d9..9b54b985d2 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -563,7 +563,7 @@ ], "grain": "household", "nonnegative_outputs": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", "operations": [ { "adult_minimum_age": 16, @@ -573,7 +573,7 @@ "conservation": "exact_total", "copies_column": "cgt_support_copies", "copy_index_column": "cgt_support_copy_index", - "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight); every copy carries weight / copies", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies", "declared_factor": 1, "expected_support_mass": 145000, "flag_column": "household_is_cgt_support_copy", diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 20fa57da78..fbf4c2979f 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"48b7648953aabc721c6672038770b18a1ff7c308d71bd982770b8a3fe259280f","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"85696cd009ce349a132f753c55cb2168ed2f548de9f628ce74a71e99a0e9e6ba","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"29386196b3832b90f7037593ab46b47af254b6153b544225f6e38f492e7103bc","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"48b7648953aabc721c6672038770b18a1ff7c308d71bd982770b8a3fe259280f","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"85696cd009ce349a132f753c55cb2168ed2f548de9f628ce74a71e99a0e9e6ba","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"fec5bddc81b70da31dcf9a301405049b71b11113bcdd2c2c5950efc8e3a4dc18","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 9ce81b7e27..ecc4d33b08 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -844,22 +844,24 @@ def _e8_support_split( flagged copies (the pre-clone rows carrying ``cgt_support_copies`` above one), each family's count is ``ceil(K x w / maximum_copy_weight)``, ``K`` the geographic clone count the stage read from the frame (microcosm#931; - the recompute reads it from the pre-split table the same way), exactly at - ``K = 1`` and at ``K > 1`` to the family weights' relative tolerance of - 1e-9 (the fold re-sums ``n`` copies of a clone's share, which drifts by - more ulps than the count's snap absorbs; the receipt's - ``copies_recomputed`` stays the exact recount, so at ``K > 1`` it can - exceed ``copies`` by one for each family at an exact multiple whose - re-summed weight lands above it, while the count check passes), every - member of a family - carries ``w / n`` at stage time (its clone pair sum, since the clone runs - afterwards), the copies of each root are indexed ``1..n-1`` without gaps - and carry the root's count, no count is below one, and the stage's mass - record conserves the total. The recompute is repeated on permuted person - and household tables and must reproduce the divided set and the counts - exactly. A selected household light enough for a single copy leaves no - trace in the artifact, so only families are compared. Problems land under - ``support_split_*`` keys. + the recompute reads it from the pre-split table the same way), to the + family weights' relative tolerance of 1e-9 at every ``K``. The fold + re-sums the ``n`` copies, which can land ulps from ``w``: at ``K = 1`` + for a weight a few ulps above a multiple of the maximum, or where the + stage's exact-total correction fell on a copy, and at ``K > 1`` by more + than the count's snap absorbs. So on an exact multiple the check accepts + ``n`` or ``n + 1``, and a count moved there is left to the flag and + family-weight checks. The receipt's ``copies_recomputed`` stays the exact + recount, so it can differ from ``copies`` by one for each family whose + re-summed weight lands across a multiple, while the count check passes. + Every member of a family carries ``w / n`` at stage time (its clone pair + sum, since the clone runs afterwards), the copies of each root are + indexed ``1..n-1`` without gaps and carry the root's count, no count is + below one, and the stage's mass record conserves the total. The + recompute is repeated on permuted person and household tables and must + reproduce the divided set and the counts exactly. A selected household + light enough for a single copy leaves no trace in the artifact, so only + families are compared. Problems land under ``support_split_*`` keys. """ from microcosm.build.uk_runtime.cgt_support import ( @@ -1052,25 +1054,24 @@ def recompute(person_t, household_t, weights_t): } agreed = (recomputed_family & stored_family).to_numpy() stored_agreed = stored_count.to_numpy()[agreed] - if clone_count == 1: - count_off = int((original["copies"].to_numpy()[agreed] != stored_agreed).sum()) - else: - # At K > 1 the fold re-sums n copies of a K-th share, which drifts by - # more ulps than the count's K-ulp snap absorbs, so an exact recount - # would flag a family the stage counted right (an exact multiple of 60 - # at K = 11, for one). The count is held to the family weights' - # tolerance instead: the stored n must lie between the counts at the - # folded weight times 1 - 1e-9 and 1 + 1e-9 (microcosm#931 review). - agreed_weights = pre_weights[agreed] - lowest, highest = ( - cgt_support_copy_counts( - agreed_weights * factor, - CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, - clone_count=clone_count, - ) - for factor in (1.0 - 1e-9, 1.0 + 1e-9) + # The fold re-sums the n copies, which can land ulps from w: at K = 1 for + # a weight a few ulps above a multiple of 60, or where the exact-total + # correction fell on a copy, and at K > 1 by more than the count's K-ulp + # snap absorbs (an exact multiple of 60 at K = 11, for one). An exact + # recount would flag a family the stage counted right, so at every K the + # count is held to the family weights' tolerance instead: the stored n + # must lie between the counts at the folded weight times 1 - 1e-9 and + # 1 + 1e-9 (microcosm#931 review). + agreed_weights = pre_weights[agreed] + lowest, highest = ( + cgt_support_copy_counts( + agreed_weights * factor, + CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + clone_count=clone_count, ) - count_off = int(((stored_agreed < lowest) | (stored_agreed > highest)).sum()) + for factor in (1.0 - 1e-9, 1.0 + 1e-9) + ) + count_off = int(((stored_agreed < lowest) | (stored_agreed > highest)).sum()) if count_off: problems["support_split_copies_stored"] = count_off @@ -1785,8 +1786,10 @@ def plans_equal(left: pd.Series, right: pd.Series) -> bool: "pair sum against w / n at rtol 1e-9 / atol 1e-6 (the fold and " "the anchor each cost one float rounding generation); " "support-split mass record: new_total against old_total at rtol " - "1e-9; support-split families, copy counts, flags and " - "student_loan_plan: exact equality; band-donor seats, bands and " + "1e-9; support-split copy counts: between the counts at the " + "folded family weight times 1 - 1e-9 and 1 + 1e-9; support-split " + "families, flags and student_loan_plan: exact equality; " + "band-donor seats, bands and " "carriers: exact equality; band-donor weights within a band and " "against the rerun: rtol 1e-9 / atol 1e-6; band-donor mass " "record: new_total against old_total at rtol 1e-9 with declared " From 4ba334a99f8ef39243518b8fd0ffdae6805431cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:16:42 +0000 Subject: [PATCH 024/108] Fold the high-value copies onto their roots and check E5 where the uprating left it (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1b decoders review found two identity-tool gaps on the synthetic H2 oracle frame. As María ruled on 8 Nov, they land as a tool-only commit before the arms. Both reproduce with origin/main's code and with the parent's. - E5's stored-column check re-applies the regional uprating (stage 25) to the stored values as a fixed point. regional_home_values (stage 27) then rewrites main_residence_value and property_wealth, so E5 was comparing stage 27's values. On the oracle, every mismatching row is one that stage 27 rewrote: 284 of 394 scoped rows on main, 294 of 404 at the parent. Stage 27 keeps the value it replaced in main_residence_value_rank_basis and moves the wealth by the value's change. The scoping now reverses that rewrite for every stage before 27. With the reversal, the mismatches drop to zero on both frames, so the stages are right and the tool was reading the wrong stage's output. - _STACKING_STAGE_BY_FLAG had no entry for household_is_high_value_support_copy (stage 26). The was_wealth scope kept the copies (105 of 394 rows on main, 115 of 404 at the parent), and E4's FRS-only frame kept any copy of an FRS household. The flag joins the map. Dropping the layer folds the copies onto their roots by family, after the CGT folds: each copy adds its w/n back, so the root returns to its pre-split weight, to rounding. A new decoder reads the copies at the copy-id table's high_value_support field. It refuses, by count, a copy without a root, a flag or index that disagrees with the id, and a family whose size disagrees with high_value_support_copies. Dropping the copies while a CGT layer stays in scope is refused. On the oracle every high-value root is an SPI row, so the copies never reach the uprating's FRS-base mean there. The new tests use FRS roots, where they do. What moves: tool verdicts only. On the oracle, through main() and the H5 loader, E5 fails with the parent's tool and passes now. E4 reads the same frame as before. The E6, E7 and E9 receipts are identical, and E8 stops at the same pre-existing error on the fixture with both tools. On a spine whose high-value roots include FRS households, E4's scope and weights change too. Nothing hashes the tool, so no stage, value, node key or generated file moves. The source_stages projection and coverage manifest checks pass unchanged. Tests: 12 new tests on frames the real stages build (stages 25, 26 and 27, then the CGT support split and the clone): - the was_wealth and FRS-only scopes drop every copy and give each root its pre-split weight; - E5 is green on that chain, red with either half of the old scope, and red when an uprated value is moved off its fixed point before stage 26; - the tool's entry point passes on that chain and fails on the moved value; - the reversal restores stage 26's values; - the lineage refusals print counts only. 9 of the 12 fail against the parent's tool. The 3 that pass are the mutation cases, which are red with both tools. The 9 touched engine-free files pass (218 tests). Ruff is clean. Co-Authored-By: Claude Opus 5.5 --- ...k-identity-tool-high-value-copies.fixed.md | 5 + .../uk/test_uk_identity_stability_receipts.py | 307 +++++++++++++++++- tools/verify_uk_identity_stability.py | 273 +++++++++++++++- 3 files changed, 560 insertions(+), 25 deletions(-) create mode 100644 changelog.d/931-uk-identity-tool-high-value-copies.fixed.md diff --git a/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md b/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md new file mode 100644 index 0000000000..933619afbd --- /dev/null +++ b/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md @@ -0,0 +1,5 @@ +The UK identity tool (`tools/verify_uk_identity_stability.py`) now scopes its receipts past the high-value home copies, and E5 checks the regional uprating on the values that stage left (#931). + +- **The copies.** The tool's stacking map gains `household_is_high_value_support_copy`, the flag of `high_value_home_support` (stage 26, microcosm#1240). The receipts that scope to an earlier stage now drop the copies and every CGT row made from them: E5's wealth block and E4's FRS-only frame. The copies' mass folds back by family. After the CGT folds, each copy's weight is added onto its root, so the root is back at its weight before the split. No uniform factor could do this, since only the top value stratum is divided. The fold decodes each copy at the copy-id table's `high_value_support` field. It refuses, by count, a copy without a root, a flag that disagrees with the stored index, an index that disagrees with the id, and a family whose decoded size disagrees with `high_value_support_copies`. Dropping the copies while a CGT layer stays in scope is refused. +- **E5's stored values.** `regional_home_values` (stage 27, microcosm#1112) rewrites `main_residence_value` and `property_wealth` after the uprating, so E5's fixed-point check of the uprating was comparing stage 27's values. The scoping now reverses that rewrite for every stage before stage 27. The value comes back from `main_residence_value_rank_basis`, which the stage keeps, and the wealth moves back by the same change. An artifact without the rank basis, such as a release, is read as before. +- **What moves.** Tool verdicts only. On the synthetic H2 oracle frame, E5's stored-column check fails with the parent's tool, as it does on origin/main, and passes now. The `was_wealth` scope drops the 115 high-value copies (of 404 rows) it kept. On that frame every high-value root is an SPI row, which E4 drops anyway, so E4 reads the same frame as before, and the E6, E7 and E9 receipts are identical. On a spine whose high-value roots include FRS households, E4's scope and weights change too. No stage, value, node key or generated file moves, since nothing hashes the tool. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 447dedf9de..4227133ab3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -41,6 +41,10 @@ UKETBServicesStageTransform, allocate_nhs_by_age_gender, ) +from microcosm.build.uk_runtime.household_lineage import ( + HIGH_VALUE_SUPPORT_COPY_COLUMN, + HIGH_VALUE_SUPPORT_INDEX_COLUMN, +) from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.frame import WeightKind from test_support.microcosm_build.uk_cgt_support import ( @@ -687,6 +691,35 @@ def test_only_layers_stacked_after_a_stage_are_stripped(self) -> None: *expected_later, ] + def test_the_high_value_copies_are_stripped_from_the_stages_before_them( + self, + ) -> None: + # Stage 26's copies (microcosm#1240) are stacked after the wealth + # stages and before the travel block. + tool = _load_tool() + household = pd.DataFrame( + { + "household_is_spi_synthetic": [False], + HIGH_VALUE_SUPPORT_COPY_COLUMN: [False], + "household_is_capital_gains_clone": [False], + "household_is_cgt_support_copy": [False], + } + ) + cgt_layers = [ + "household_is_capital_gains_clone", + "household_is_cgt_support_copy", + ] + assert tool._flags_stacked_after("was_wealth", household) == [ + HIGH_VALUE_SUPPORT_COPY_COLUMN, + *cgt_layers, + ] + assert tool._flags_stacked_after("nts_bus_travel", household) == cgt_layers + assert tool._flags_stacked_after("frs_take_up", household) == [ + "household_is_spi_synthetic", + HIGH_VALUE_SUPPORT_COPY_COLUMN, + *cgt_layers, + ] + def test_e5_receipt_covers_the_spi_rows_the_stage_imputed(self) -> None: from microcosm.build.uk_runtime.regional_uprating import ( uprate_household_property_by_region, @@ -1318,10 +1351,6 @@ def _lineage_household(household: pd.DataFrame) -> pd.DataFrame: HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, ) from microcosm.build.uk_runtime.cgt_support import CGT_SUPPORT_COPY_INDEX_COLUMN - from microcosm.build.uk_runtime.household_lineage import ( - HIGH_VALUE_SUPPORT_COPY_COLUMN, - HIGH_VALUE_SUPPORT_INDEX_COLUMN, - ) from test_support.microcosm_build.uk_clone_first_frames import COPY_PATH_COLUMNS lineage = household.drop(columns=list(COPY_PATH_COLUMNS)) @@ -1380,10 +1409,6 @@ def test_the_gate_holds_each_stored_ordinal_to_the_ids_exactly(self) -> None: # The SPI channel's ordinal (copy 1, band donor 2) and the high-value # copy's are checked against the id only here; a nonzero-marker reading # would pass a band donor recorded as the SPI copy. - from microcosm.build.uk_runtime.household_lineage import ( - HIGH_VALUE_SUPPORT_INDEX_COLUMN, - ) - tool = _load_tool() household = _lineage_household(_chain("table").household) for column, stored, tampered_to in ( @@ -1670,3 +1695,269 @@ def test_the_arms_the_stage_built_decode_and_receipt_green(self) -> None: assert problems == {} assert pairs["anchored"] is True and pairs["pairs"] == 11 assert support["copies"] == 5 and support["families"] == 3 + + +# --- the high-value home copies and the regional home values ----------------- +# +# The split scenario's six one-person households, all FRS-base owners, through +# the real stages from the regional uprating on. The uprating (stage 25) sets +# each region's unweighted mean of base owners' values to its public price; +# the high-value home support (stage 26, microcosm#1240) divides the top of +# each region, at stratum 0.6, into three rows at a third of the weight (id 5 +# in Wales, ids 4 and 6 in London); the regional home values (stage 27, +# microcosm#1112) read every value off the region's knots and keep the value +# they replaced. With ``cgt`` the CGT support split and the clone follow. + +_HOME_VALUES = [100_000.0, 200_000.0, 300_000.0, 400_000.0, 500_000.0, 600_000.0] +_OTHER_PROPERTY = [0.0, 50_000.0, 10_000.0, 0.0, 120_000.0, 30_000.0] + + +def _home_values_chain(*, cgt: bool, tamper: bool = False) -> SimpleNamespace: + """The scenario before stage 26, after it, after stage 27, and at the end. + + With ``tamper`` the first household's uprated value and wealth move off + the uprating's factor before stage 26, as an uprating that stored a value + off its fixed point would leave them. + """ + + from microcosm.build.uk_runtime.high_value_home_support import ( + split_high_value_support_households, + ) + from microcosm.build.uk_runtime.regional_home_values import ( + assign_regional_home_values, + ) + from microcosm.build.uk_runtime.regional_uprating import ( + load_regional_land_values_resource, + uprate_household_property_by_region, + ) + from test_support.microcosm_build.uk_cgt_support import support_tables + + person, benunit, household, weights = support_tables( + weights=_SPLIT_WEIGHTS, wealth=_SPLIT_WEALTH, incomes=_SPLIT_INCOMES + ) + household["tenure_type"] = "OWNED_OUTRIGHT" + household["main_residence_value"] = _HOME_VALUES + household["property_wealth"] = np.add(_HOME_VALUES, _OTHER_PROPERTY) + household["source_household_id"] = household["household_id"] + uprated = uprate_household_property_by_region( + household, load_regional_land_values_resource() + ) + if tamper: + uprated.loc[0, ["main_residence_value", "property_wealth"]] *= 1.25 + pre_split = uk_national_frame( + person=person, + benunit=benunit, + household=uprated, + household_weights=weights, + time_period="2024", + ) + split = split_high_value_support_households( + pre_split, stratum_lower=0.6, copies=3 + ).frame + valued = assign_regional_home_values(split).frame + final = valued + if cgt: + final = clone_cgt_incidence( + split_cgt_support_households( + valued, + distribution=support_distribution(_SPLIT_TOP_CELLS), + parameters=PARAMETERS, + clone_count=1, + ).frame, + distribution=load_advani_summers_distribution(), + ).frame + return SimpleNamespace(pre_split=pre_split, split=split, valued=valued, final=final) + + +def _cgt_layers(frame) -> list[str]: + household = frame.table("household") + return [ + flag + for flag in (HOUSEHOLD_IS_CGT_CLONE, HOUSEHOLD_IS_CGT_SUPPORT_COPY) + if flag in household.columns + ] + + +class TestHighValueCopiesAndRegionalHomeValues: + """E5 reads the uprating where stage 25 left it, on the rows it saw.""" + + @pytest.mark.parametrize("cgt", [False, True]) + def test_the_wealth_scope_folds_every_copy_onto_its_root(self, cgt: bool) -> None: + tool = _load_tool() + chain = _home_values_chain(cgt=cgt) + pre_split = chain.pre_split.weights_for("household").values + # The split leaves each root a third of its weight, its two copies + # the other thirds. + split_household = chain.split.table("household") + families = split_household["high_value_support_copies"].to_numpy() + assert families.tolist() == [1, 1, 1, 3, 3, 3] + [3] * 6 + np.testing.assert_allclose( + chain.split.weights_for("household").values[:6], + pre_split / families[:6], + rtol=1e-12, + ) + for scoped in ( + tool._frame_as_stage_saw(chain.final, "was_wealth"), + tool._frs_only_frame(chain.final), + ): + household = scoped.table("household") + assert household["household_id"].tolist() == [1, 2, 3, 4, 5, 6] + assert not household[HIGH_VALUE_SUPPORT_COPY_COLUMN].any() + assert scoped.table("person")["person_id"].tolist() == [1, 2, 3, 4, 5, 6] + # Each root's copies add their thirds back: its pre-split mass. + np.testing.assert_allclose( + scoped.weights_for("household").values, pre_split, rtol=1e-12 + ) + # The travel block runs after the split: its scope keeps the copies. + travel = tool._frame_as_stage_saw(chain.final, "nts_bus_travel") + assert int(travel.table("household")[HIGH_VALUE_SUPPORT_COPY_COLUMN].sum()) == 6 + + @pytest.mark.parametrize("cgt", [False, True]) + def test_e5_holds_the_uprating_where_it_left_the_values(self, cgt: bool) -> None: + tool = _load_tool() + final = _home_values_chain(cgt=cgt).final + receipt = tool.e5_identity_receipt( + tool._frame_as_stage_saw(final, "was_wealth"), permutation_seed=7 + ) + assert receipt["columns_by_entity"]["household"] == [ + "main_residence_value", + "property_wealth", + ] + assert receipt["identical_under_permutation"] is True + assert receipt["matches_stored_columns"] is True + # Either half of the scope alone stays red: the values stage 27 + # stored, or the copies, FRS-base owners that enter the uprating's + # mean when they stay in scope. + stored_values = tool._drop_stacked_layers( + final, [HIGH_VALUE_SUPPORT_COPY_COLUMN, *_cgt_layers(final)] + ) + copies_kept = tool._drop_stacked_layers( + tool._pre_regional_home_values_frame(final), _cgt_layers(final) + ) + for partial in (stored_values, copies_kept): + red = tool.e5_identity_receipt(partial, permutation_seed=7) + assert red["matches_stored_columns"] is False + assert red["stored_column_mismatches"] == { + "household": ["main_residence_value", "property_wealth"] + } + + @pytest.mark.parametrize("cgt", [False, True]) + def test_e5_reports_an_uprated_value_off_its_fixed_point(self, cgt: bool) -> None: + tool = _load_tool() + final = _home_values_chain(cgt=cgt, tamper=True).final + receipt = tool.e5_identity_receipt( + tool._frame_as_stage_saw(final, "was_wealth"), permutation_seed=7 + ) + assert receipt["identical_under_permutation"] is True + assert receipt["matches_stored_columns"] is False + assert receipt["stored_column_mismatches"] == { + "household": ["main_residence_value", "property_wealth"] + } + + @pytest.mark.parametrize("tamper", [False, True]) + def test_the_tool_entry_point_receipts_e5( + self, tamper: bool, monkeypatch, tmp_path + ) -> None: + import json + import sys + + tool = _load_tool() + final = _home_values_chain(cgt=True, tamper=tamper).final + monkeypatch.setattr(tool, "load_uk_national_frame", lambda path: (final, {})) + output = tmp_path / "receipt.json" + monkeypatch.setattr( + sys, + "argv", + [ + "verify_uk_identity_stability.py", + "--input-h5", + str(tmp_path / "spine.h5"), + "--output", + str(output), + "--check", + "e5", + ], + ) + assert tool.main() == (1 if tamper else 0) + assert json.loads(output.read_text())["matches_stored_columns"] is not tamper + + def test_the_reversal_restores_what_stage_27_replaced(self) -> None: + from microcosm.build.uk_runtime.regional_home_values import RANK_BASIS_COLUMN + + tool = _load_tool() + chain = _home_values_chain(cgt=False) + before = chain.split.table("household") + after = chain.valued.table("household") + assert ( + after["main_residence_value"].to_numpy() + != before["main_residence_value"].to_numpy() + ).all() + reversed_frame = tool._pre_regional_home_values_frame(chain.valued) + restored = reversed_frame.table("household") + assert RANK_BASIS_COLUMN not in restored.columns + np.testing.assert_array_equal( + restored["main_residence_value"].to_numpy(), + before["main_residence_value"].to_numpy(), + ) + np.testing.assert_allclose( + restored["property_wealth"].to_numpy(), + before["property_wealth"].to_numpy(), + rtol=1e-12, + ) + # Nothing to take back on a frame without the rank basis. + assert tool._pre_regional_home_values_frame(reversed_frame) is reversed_frame + assert tool._pre_regional_home_values_frame(chain.split) is chain.split + + def test_dropping_the_copies_under_a_kept_cgt_layer_is_refused(self) -> None: + tool = _load_tool() + final = _home_values_chain(cgt=True).final + with pytest.raises(ValueError, match="requires dropping the CGT layers"): + tool._drop_stacked_layers(final, [HIGH_VALUE_SUPPORT_COPY_COLUMN]) + + def test_a_copy_off_its_lineage_fails_closed_by_count(self) -> None: + tool = _load_tool() + household = _home_values_chain(cgt=True).final.table("household") + lineage = tool._high_value_copy_lineage(household) + assert len(lineage.copy_positions) == 6 + ids = household["household_id"].to_numpy() + # The roots' ids are single digits, like the counts; every copy's is + # long enough to show if a message printed it. + copy_ids = ids[ids >= copy_id_table.UK_COPY_ID_BASE_RADIX] + first_copy = int(lineage.copy_positions[0]) + root = int(lineage.root_positions[0]) + + index_moved = household.copy() + index_moved.loc[first_copy, HIGH_VALUE_SUPPORT_INDEX_COLUMN] = 3 - int( + lineage.copy_index[0] + ) + unflagged = household.copy() + unflagged.loc[first_copy, HIGH_VALUE_SUPPORT_COPY_COLUMN] = False + copy_missing = household.drop(index=first_copy).reset_index(drop=True) + root_missing = household.drop(index=root).reset_index(drop=True) + for tampered, message in ( + ( + index_moved, + "High-value home copy index stored on 1 copy(ies) disagrees with " + "the id scheme (the id's ordinal in the copy-id table's " + "'high_value_support' field).", + ), + ( + unflagged, + "High-value home copy flag and stored copy index disagree on 1 " + "household(s).", + ), + ( + copy_missing, + "'high_value_support_copies' stored on 1 household(s) disagrees " + "with the copies their ids decode to.", + ), + ( + root_missing, + "2 high-value home copy(ies) without a root household at the " + "declared radix of the copy-id table's 'high_value_support' field.", + ), + ): + with pytest.raises(ValueError) as caught: + tool._high_value_copy_lineage(tampered) + assert str(caught.value) == message + assert not any(str(int(value)) in str(caught.value) for value in copy_ids) diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index ecc4d33b08..8158641b44 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -299,7 +299,10 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: # The caller scopes the frame to the population the wealth and uprating # stages saw (``_frame_as_stage_saw``): the SPI support rows run before - # them and are imputed there, while the CGT layers are stacked later. + # them and are imputed there, while the high-value home copies and the + # CGT layers are stacked later. The same scoping takes back the regional + # home values' later rewrite of both property columns, so the uprating's + # fixed point is read off the values the uprating stage left. person = frame.table("person") benunit = frame.table("benunit") household = frame.table("household") @@ -362,8 +365,9 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: "waterfall re-allocation; stored-column cross-check: rtol 1e-12 " "/ atol 1e-6 GBP for both layers (re-applying uprating to the " "uprated fixed point and re-summing allocations each cost one " - "float rounding generation); corporate_wealth fold not " - "reconstructible from the artifact (components consumed) - " + "float rounding generation, and the scoping's reversal of the " + "regional home values' wealth move one more); corporate_wealth " + "fold not reconstructible from the artifact (components consumed) - " "unit-tested only" ), "columns_by_entity": { @@ -2014,10 +2018,14 @@ def main() -> int: ) elif args.check == "e5": # E5's wealth stages run after the SPI support rows are stacked and - # before the CGT layers. The regional property uprating's mean is over - # FRS-base owners, so a frame still carrying the later CGT copies - # shifts that denominator and the recomputation stops matching what - # the stage stored. Scope to the population the stage saw. + # before the high-value home copies and the CGT layers. The regional + # property uprating's mean is over FRS-base owners, so a frame still + # carrying the later copies shifts that denominator and the + # recomputation stops matching what the stage stored. Scope to the + # population the stage saw. regional_home_values then rewrites both + # property columns, so the fixed point holds only on the values the + # uprating left, which the scoping restores from that stage's rank + # basis. receipt = e5_identity_receipt( _frame_as_stage_saw(frame, _E5_FIRST_STAGE), permutation_seed=args.permutation_seed, @@ -2084,6 +2092,9 @@ def main() -> int: _STACKING_STAGE_BY_FLAG = { # #717 SPI support channel; the #1006 income band donors carry the same flag. "household_is_spi_synthetic": "spi_support_channel", + # microcosm#1240: the high-value home copies; every CGT layer copies them + # with this flag, so dropping this layer drops their CGT rows too. + "household_is_high_value_support_copy": "high_value_home_support", "household_is_capital_gains_clone": "cgt_incidence_clone", # E8 clone "household_is_cgt_support_copy": "cgt_support_split", # E8 support copies # microcosm#1063: the residential split's arms carry their source's clone @@ -2103,10 +2114,10 @@ def main() -> int: #: The weight restoration is driven by what the *artifact* actually contains #: rather than by the committed roster: a spine built before a stacking stage #: existed never had that stage's mass factor applied, and dividing it out -#: anyway skews the comparison in the opposite direction. The CGT clone and -#: the CGT support split are deliberately absent: neither is a uniform -#: factor, so their mass is restored by fold instead -#: (``_pre_split_household_weights``). +#: anyway skews the comparison in the opposite direction. The CGT clone, the +#: CGT support split and the high-value home split are deliberately absent: +#: none is a uniform factor, so their mass is restored by fold instead +#: (``_pre_split_household_weights``, ``_pre_high_value_household_weights``). _MASS_STAGE_BY_FLAG = { "household_is_spi_synthetic": "spi_support_channel", } @@ -2496,6 +2507,150 @@ def _pre_split_household_weights(frame) -> np.ndarray: return weights +class _HighValueCopyLineage(NamedTuple): + """Where every high-value home copy sits and which root it came from.""" + + copy_positions: np.ndarray + root_positions: np.ndarray + copy_index: np.ndarray + + +def _high_value_copy_lineage(household: pd.DataFrame) -> _HighValueCopyLineage: + """Every high-value home copy's position, its root's and its index ``k``. + + ``high_value_home_support`` (microcosm#1240) divides each owner household + in the top value stratum of its region into ``high_value_support_copies`` + rows: copy ``k`` of root ``r`` carries ``r + k x R`` on every id, ``R`` the + declared radix of the copy-id table's ``high_value_support`` field + (microcosm#931), and ``k`` in ``high_value_support_copy_index`` (0 on + roots and unsplit households). Every CGT layer runs after it and copies + the flag with the row, so the copies and roots read here are the rows no + CGT layer created. A copy without a root, a flag that disagrees with the + stored index, an index that disagrees with the id, or a family whose + decoded size disagrees with the stored ``high_value_support_copies`` + fails closed. The refusals print counts, never an id. + """ + + from microcosm.build.uk_runtime.cgt_residential_split import ( + HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, + ) + from microcosm.build.uk_runtime.cgt_structure import HOUSEHOLD_IS_CGT_CLONE + from microcosm.build.uk_runtime.cgt_support import HOUSEHOLD_IS_CGT_SUPPORT_COPY + from microcosm.build.uk_runtime.high_value_home_support import ( + HIGH_VALUE_SUPPORT_COPIES_COLUMN, + HIGH_VALUE_SUPPORT_COPY_ID_FIELD, + ) + from microcosm.build.uk_runtime.household_lineage import ( + HIGH_VALUE_SUPPORT_COPY_COLUMN, + HIGH_VALUE_SUPPORT_INDEX_COLUMN, + ) + + household_ids = _exact_household_ids(household, what="High-value home lineage") + if not pd.Index(household_ids).is_unique: + raise ValueError("High-value home lineage requires unique household ids.") + missing = [ + column + for column in ( + HIGH_VALUE_SUPPORT_INDEX_COLUMN, + HIGH_VALUE_SUPPORT_COPIES_COLUMN, + ) + if column not in household.columns + ] + if missing: + raise ValueError(f"High-value home lineage requires {missing}.") + is_copy = household[HIGH_VALUE_SUPPORT_COPY_COLUMN].to_numpy(dtype=bool) + stored_index = ( + pd.to_numeric(household[HIGH_VALUE_SUPPORT_INDEX_COLUMN], errors="raise") + .astype("int64") + .to_numpy() + ) + family_size = ( + pd.to_numeric(household[HIGH_VALUE_SUPPORT_COPIES_COLUMN], errors="raise") + .astype("int64") + .to_numpy() + ) + flag_disagreements = int(((stored_index != 0) != is_copy).sum()) + if flag_disagreements: + raise ValueError( + "High-value home copy flag and stored copy index disagree on " + f"{flag_disagreements} household(s)." + ) + created_later = np.zeros(len(household), dtype=bool) + for flag in ( + HOUSEHOLD_IS_CGT_SUPPORT_COPY, + HOUSEHOLD_IS_CGT_CLONE, + HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, + ): + if flag in household.columns: + created_later |= household[flag].to_numpy(dtype=bool) + copy_positions = np.flatnonzero(is_copy & ~created_later) + copy_index, root_ids = _copy_ordinals_and_parents( + household_ids[copy_positions], HIGH_VALUE_SUPPORT_COPY_ID_FIELD + ) + root_positions, orphaned = _parent_positions( + household_ids, copy_index, root_ids, eligible=~is_copy & ~created_later + ) + if orphaned: + raise ValueError( + f"{orphaned} high-value home copy(ies) without a root household at the " + "declared radix of the copy-id table's " + f"{HIGH_VALUE_SUPPORT_COPY_ID_FIELD!r} field." + ) + scheme_disagreements = int((stored_index[copy_positions] != copy_index).sum()) + if scheme_disagreements: + raise ValueError( + f"High-value home copy index stored on {scheme_disagreements} " + "copy(ies) disagrees with the id scheme (the id's ordinal in the " + f"copy-id table's {HIGH_VALUE_SUPPORT_COPY_ID_FIELD!r} field)." + ) + # Each root's family is the root and the copies decoded onto it; a copy + # missing from the artifact would leave its root's fold short. + decoded_size = np.bincount(root_positions, minlength=len(household)) + 1 + roots = ~is_copy & ~created_later + family_disagreements = int( + (decoded_size[roots] != family_size[roots]).sum() + + (family_size[copy_positions] != family_size[root_positions]).sum() + ) + if family_disagreements: + raise ValueError( + f"{HIGH_VALUE_SUPPORT_COPIES_COLUMN!r} stored on " + f"{family_disagreements} household(s) disagrees with the copies " + "their ids decode to." + ) + return _HighValueCopyLineage( + copy_positions=copy_positions, + root_positions=root_positions, + copy_index=copy_index, + ) + + +def _pre_high_value_household_weights(frame) -> np.ndarray: + """Household weights with every high-value home copy folded onto its root. + + ``high_value_home_support`` (microcosm#1240) divides each owner household + in the top value stratum of its region into ``n`` rows at ``w / n``, so + adding each copy's weight back onto its root restores ``w``: no uniform + divisor could, since only the stratum is divided. Every CGT layer is + stacked after this split, so those layers fold first + (``_pre_split_household_weights``), which leaves each copy the weight + this split gave it, and then the copies fold. An artifact without the + copy layer returns the CGT folds alone. Copies keep their own weights in + the returned vector; the scoping that calls this drops those rows. + """ + + from microcosm.build.uk_runtime.household_lineage import ( + HIGH_VALUE_SUPPORT_COPY_COLUMN, + ) + + weights = _pre_split_household_weights(frame) + household = frame.table("household") + if HIGH_VALUE_SUPPORT_COPY_COLUMN not in household.columns: + return weights + lineage = _high_value_copy_lineage(household) + np.add.at(weights, lineage.root_positions, weights[lineage.copy_positions]) + return weights + + def _roster_positions() -> dict[str, int]: """Stage positions in the committed source-stage roster.""" @@ -2583,9 +2738,10 @@ def _frs_only_frame(frame): On the E8 roster this leaves the 16,288 raw FRS households, which is the population the pre-stacking stages actually drew for. Their weights fold each clone back onto its original (exact before and after the #970 - anchor) and each CGT support copy back onto its root, so every surviving - row carries its pre-split weight. Every pre-stacking stage precedes - salary_sacrifice, so the conversion's pay rewrite is reversed too. + anchor), and each CGT support copy and each high-value home copy back + onto its root, so every surviving row carries its pre-split weight. + Every pre-stacking stage precedes salary_sacrifice, so the conversion's + pay rewrite is reversed too. """ frame = _pre_salary_sacrifice_frame(frame) @@ -2599,12 +2755,16 @@ def _frame_as_stage_saw(frame, stage: str): """Scope the artifact to the rows present when ``stage`` ran. A stage before ``salary_sacrifice`` also saw every converted record's - pay and employee contribution before the conversion rewrote them. + pay and employee contribution before the conversion rewrote them, and a + stage before ``regional_home_values`` every owner's home value and + property wealth before that stage rewrote them. """ positions = _roster_positions() if positions[stage] < positions[_SALARY_SACRIFICE_STAGE]: frame = _pre_salary_sacrifice_frame(frame) + if positions[stage] < positions[_REGIONAL_HOME_VALUES_STAGE]: + frame = _pre_regional_home_values_frame(frame) return _drop_stacked_layers( frame, _flags_stacked_after(stage, frame.table("household")) ) @@ -2702,12 +2862,71 @@ def _pre_salary_sacrifice_frame(frame): ) +#: The stage that rewrites owners' home values in place (microcosm#1112). +_REGIONAL_HOME_VALUES_STAGE = "regional_home_values" + + +def _pre_regional_home_values_frame(frame): + """The artifact with the regional home values' rewrite reversed. + + ``regional_home_values`` gives England-and-Wales owners the value their + region's Land Registry distribution holds at their rank, keeps the value + it replaced on every household in ``main_residence_value_rank_basis`` + (its only writer) and moves ``property_wealth`` by the value's change, + floored at zero. So the rewrite reverses: the value back from the rank + basis, exactly, and the wealth moved back by the same change, at one + rounding. ``was_wealth`` derives ``property_wealth`` as a sum of property + values that includes ``main_residence_value``, so the moved wealth is at + least the new value and the floor binds only where a frame broke that + sum; there the wealth it replaced is lost, and the reversal gives the + largest wealth the floor allows. The rank basis is dropped, since no + earlier stage had it. An artifact without it (built before the stage, or + a release, whose boundary drops it with ``property_wealth``) is returned + as it is. + """ + + from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + ) + from microcosm.build.uk_runtime.regional_home_values import RANK_BASIS_COLUMN + + household = frame.table("household") + if RANK_BASIS_COLUMN not in household.columns: + return frame + household = household.copy() + basis = pd.to_numeric(household[RANK_BASIS_COLUMN], errors="raise").to_numpy( + dtype=float + ) + value = pd.to_numeric(household["main_residence_value"], errors="raise").to_numpy( + dtype=float + ) + if "property_wealth" in household.columns: + wealth = pd.to_numeric(household["property_wealth"], errors="raise").to_numpy( + dtype=float + ) + household["property_wealth"] = wealth - (value - basis) + household["main_residence_value"] = basis + return uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=household.drop(columns=[RANK_BASIS_COLUMN]), + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + + def _drop_stacked_layers(frame, flags: Sequence[str]): from microcosm.build.uk_runtime.cgt_residential_split import ( HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, ) from microcosm.build.uk_runtime.cgt_structure import HOUSEHOLD_IS_CGT_CLONE from microcosm.build.uk_runtime.cgt_support import HOUSEHOLD_IS_CGT_SUPPORT_COPY + from microcosm.build.uk_runtime.household_lineage import ( + HIGH_VALUE_SUPPORT_COPY_COLUMN, + ) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, @@ -2719,11 +2938,31 @@ def _drop_stacked_layers(frame, flags: Sequence[str]): return frame stacked = household[flags].astype(bool).any(axis=1) keep = ~stacked - # Fold created CGT rows' mass back only when their layer is the one being + # Fold created rows' mass back only when their layer is the one being # dropped: the clone by pair, the support copy by family onto its root. # The split runs before the clone, so dropping the split layer while the # artifact's clone layer stays in scope would fold clones that remain. - if HOUSEHOLD_IS_CGT_SUPPORT_COPY in flags: + # The high-value home copies also fold by family onto their roots, and + # every CGT layer is stacked after them, so dropping them requires + # dropping every CGT layer the artifact carries. + if HIGH_VALUE_SUPPORT_COPY_COLUMN in flags: + kept_later = [ + flag + for flag in ( + HOUSEHOLD_IS_CGT_SUPPORT_COPY, + HOUSEHOLD_IS_CGT_CLONE, + HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, + ) + if flag in household.columns and flag not in flags + ] + if kept_later: + raise ValueError( + "Dropping the high-value home copies requires dropping the CGT " + f"layers stacked after them; the artifact carries {kept_later} " + f"but {flags} does not name them." + ) + all_weights = _pre_high_value_household_weights(frame) + elif HOUSEHOLD_IS_CGT_SUPPORT_COPY in flags: if HOUSEHOLD_IS_CGT_CLONE in household.columns and ( HOUSEHOLD_IS_CGT_CLONE not in flags ): From 254bbe4faae1b79633acc725ed45cc653eba527b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:31:18 +0000 Subject: [PATCH 025/108] Follow up the review of the sampler, binding, fallback and tool commits (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three reviews of 2afda31fe, fe1541379, e279b69b0 and bd292ebe9. Their one must, the stage-42 correction rule, needs a ruling and is left out with the other ruling items listed below. Bindings (fe1541379): - The rule took any constant made only of strings as unable to change a value. A tuple, set or mapping of strings selects rows, columns or codes, so it is now a value, like a number. Only a single string, a function returning one column name and a record whose field types are of these kinds still stop the rule. Newly bound: tenure_constants in was_wealth, was_lisa, high_value_home_support and regional_home_values; council_tax_constants in frs_spine; rowwise_geography in lcfs_consumption; etb_vat in etb_services, which joins the counting stages through it; cgt_imputation in cgt_incidence_clone; frs_annual_income and hmrc_source_contract in hmrc_spi_income_spine; household_income in was_wealth and was_lisa. The dense population hash adds spi_support. - Where the rule stops at a name, a new test requires the name's module to be hashed by the stage or by a node upstream of it, since node keys chain on their predecessors', or the name to be listed with a reason. One stage name is listed (the geographic clone index columns, which no spine node writes, checked) and two dense ones (the LAD23 lookup's pin). - The derived check kind goes: a check is skipped only when listed with its reason, now also cgt_structure's closed-world check and spi_support's entity check. _writes_nothing, which tests each entry, also counts an augmented assignment, out=, copyto, ufunc.at, shuffle, reverse and the like as writes, and lets only a one-parameter function return an argument. A record's field types must be harmless too. - The spine root's rung sampler moves from spine_build, which no node hashes, to national_sampling (UKSampledSpineRootTransform and sample_uk_spine_root, behaviour unchanged). It declares national_sampling and frame_sampling through graph_implementation_dependencies, so the production root node hashes them. A test holds every transform spine_build installs to its stage's module, declared dependencies or a listed adapter. - The scope comment and changelog said microcosm.fit's fitters are bound by hand; three are, in two stages. They now name what stays unbound outside microcosm.build and the dense kernels the rule is not yet applied to, and the changelog names stage 18's employer pension binding. Fallback (e279b69b0), texts only: the tool's docstring, comment and tolerance policy, the receipts test class and the changelog say that within 1e-9 of a multiple of 60 the count check accepts n or n + 1, so a family split consistently into one copy more goes unreported at every K; a new test pins it. The changelog's what-moves names the tolerance policy and the country- and solve-fingerprint kernels that move, and puts the correction row about an ulp of the frame's total off w / n. Stage 26's docstring names the shared fallback as stage 42's does. Tool (bd292ebe9): the E5 reversal refuses, by count, a roster where a stage after regional_home_values declares main_residence_value or property_wealth. A test holds the stacking map to the structural stages' declared flags, with two listed exceptions. Sampler (2afda31fe): the seed comment drops the retired draws, and a test pins uk.full.sample's f100 branch, which experiments/901's row 4 now cites. Left for María: the stage-42 correction rule (on the synthetic nesting scenario no whole household can carry it at K = 7 and 13), an exact K = 1 count check, binding the unbound modules outside microcosm.build, binding cgt_structure in stages 26 and 46 instead of listing its check, and copy_rule's wording, which waits for the correction ruling. For the PR body: 2afda31fe adds a changelog fragment of type removed; fe1541379 binds employer_pension_contributions in stages 14 and 18; e279b69b0 also re-keys the spine, full, preflight and holdout gates and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. What moves: node keys only. graph_kernels.py changes, so every spine node re-keys; the 12 new bindings would re-key their 9 stages by themselves. national_sampling.py and graph_population.py change, so every graph_population kernel and the calibration and size kernels that hash it re-key. The production root node hashes the rung sampler at every rung. No value, declaration or generated file moves: the source_stages projection, coverage manifest and H2 fixture checks pass, and H2 parity passes. The identity tool's receipts change only in the tolerance policy text. Tests: 34 touched and related engine-free files pass (1,115 tests), and ruff is clean on the 11 edited Python files. In-memory mutations under scratch turn the new tests red: a dropped tenure_constants or cgt_imputation binding; a stage newly reading a collection of strings, calling an np.add.at helper in an unbound module or reading an unhashed single string; an emptied or padded names table; spi_support or rowwise_geography dropped from the dense hash; the rung sampler declaring nothing or only its own module; an unlisted source adapter; the stage-26 flag dropped from the stacking map; uk.full.sample sampling at f100; and the tool before the K = 1 tolerance on the n + 1 pin. The source_stages projection and coverage manifest checks pass, the H2 fixture regenerates byte for byte and H2 parity passes in the engine slot. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 6 +- changelog.d/931-uk-helper-bindings.changed.md | 32 +- ...k-identity-tool-high-value-copies.fixed.md | 4 +- experiments/901-uk-main-rebase-receipts.md | 7 +- .../build/uk_runtime/graph_kernels.py | 61 +- .../build/uk_runtime/graph_population.py | 11 +- .../uk_runtime/high_value_home_support.py | 5 +- .../build/uk_runtime/national_sampling.py | 126 +++- .../microcosm/build/uk_runtime/spine_build.py | 108 +--- .../engine_free/uk/test_uk_cgt_support.py | 7 +- .../tests/engine_free/uk/test_uk_frs_spine.py | 12 +- .../tests/engine_free/uk/test_uk_graph.py | 537 ++++++++++++++---- .../uk/test_uk_identity_stability_receipts.py | 113 +++- .../uk/test_uk_national_sampling.py | 38 ++ tools/verify_uk_identity_stability.py | 67 ++- 15 files changed, 882 insertions(+), 252 deletions(-) diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index 58f7cc7c33..4c564e4a86 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,6 +1,6 @@ The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. -- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. At every K its count check accepts a family's stored count within the family weights' relative tolerance of 1e-9, because folding the family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where the stage's exact-total correction fell on a copy, and at K > 1 by more than the snap absorbs. On an exact multiple of 60 the check therefore accepts n or n + 1 copies, and a count moved there is reported by the flag and family-weight checks instead. The receipt's tolerance policy says so. +- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. At every K its count check accepts a family's stored count within the family weights' relative tolerance of 1e-9, because folding the family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where the stage's exact-total correction fell on a copy, and at K > 1 by more than the snap absorbs. So for a family whose weight lies within 1e-9 of a multiple of 60, the check accepts n or n + 1 copies. A stored count moved there on its own is still reported, by the flag and family-weight checks. A family the stage split consistently into one copy more there, every member at w / (n + 1), goes unreported at every K. At K = 1 the exact check used to report it. The receipt's tolerance policy says so. - **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies". The stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. -- **Exact total.** The exact-total correction is unchanged. It lands on the heaviest unselected household, so every copy weighs bitwise w / n, except where that household cannot carry it: spi_support's shared fallback then places it, and that row can be a root or a copy, a few ulps off w / n. -- **What moves.** No value moves at K = 1. On synthetic frames, the stage's output and the identity tool's receipt are unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. At K = 1 the identity tool no longer reports a count problem for a family that folds back within the tolerance of its stored count. The stage evidence gains `clone_count`. `cgt_support_split`'s stage contract moves with its texts. Its implementation hash moves, and so does `high_value_home_support`'s, which binds `cgt_support`. The spine gates hash `stage_health` and the country spec, both of which moved, so their keys move, and with them every spine stage after the assembled gate. The coverage manifest's source digests and the H2 fixture's declarations and contract hash are regenerated. +- **Exact total.** The exact-total correction is unchanged. It lands on the heaviest unselected household, so every copy weighs bitwise w / n, except where that household cannot carry it: spi_support's shared fallback then places it, and that row can be a root or a copy. The row is then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. +- **What moves.** No value moves at K = 1. On synthetic frames, the stage's output is unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged. Its tolerance policy now names the count bracket, and at K = 1 it no longer reports a count problem for a family that folds back within the tolerance of its stored count. The stage evidence gains `clone_count`. `cgt_support_split`'s stage contract moves with its texts. Its implementation hash moves, and so does `high_value_home_support`'s, which binds `cgt_support`. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hash are regenerated. diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index a3f218020c..b1da9d6f2e 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,5 +1,29 @@ -Every UK spine stage now hashes each module its transform computes through, and the dense population nodes each module theirs do (#931). `source_hash` hashes only the files it is given, so a stage that called a function in a module its hash did not name replayed a stale cached output when that module changed. +Every UK spine stage now hashes each module in `microcosm.build` that its transform computes through, and so do the seven dense population kernels that share `_PopulationKernel`'s hash (#931). `source_hash` hashes only the files it is given, so a stage that called a function in a module its hash did not name replayed a stale cached output when that module changed. -- **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage, follows every definition the code names (annotations included, through re-exports and module attributes) into the module that defines it, within `microcosm.build`, to a fixed point, and checks that the stage's hash covers every module it meets. Five kinds of definition need no binding and are not followed: a check that returns nothing and writes nothing, a function returning one column name, a record class of undefaulted fields, a constant made only of strings, and the stage's own declaration (`country_spec`, `source_manifest` and the spec engine, whose content the node key already hashes as the stage contract). A numeric constant needs its module bound. The workspace's other packages stay outside the rule: `microcosm.graph` and `microcosm.frame` run every node, and `microcosm.fit`'s fitters are bound by hand. A second test applies the rule to the seven dense population kernels that share `_PopulationKernel`'s hash, where the spine gate admission check is listed as refusal-only, and a control test pins each way of naming a helper that the rule follows and each kind it skips. -- **What is bound.** `national_frame` joins the modules every spine stage hashes. The stage helper table gains 219 bindings of 49 modules in 48 stages, among them `property_localisation`, `cgt_structure`, `spi_support`'s exact-total helpers, `cgt_imputation`, `cgt_asset_type` and `advani_summers` in the CGT and high-value stages, and `employer_pension_contributions` in `frs_hmrc_spine_leaves`. Two groups recur: the vendored-fact reader with the chronicle feed and its schema epochs (20 stages), and `frs_spine`'s pinned tab reader with `trace` (14 stages). Because the new helpers import them, `stochastic_assignment` is bound into 8 more stages and the SPI support channel's exemption goes, the copy-id table is bound into `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`, 15 stages join the copy-id test's name-only list, and `frs_spine`, `spi_support_channel`, `spi_income_band_donors` and `uc_deduction_attributes` join the counting stages. The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input` and `gates`. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. +- **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports and module attributes. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: + - three listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; + - the stage's own declaration (`country_spec`, `source_manifest` and the spec engine), whose content the node key already hashes as the stage contract; + - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; one such name is listed. + + A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies the rule to the seven dense population kernels, with two listed names. Two more cover the transforms the production spine build installs itself. Control tests pin each way of naming a helper that the rule follows, each kind it skips, and each way a refusal-only check could write. +- **Scope.** The rule follows `microcosm.build` only. `microcosm.graph` and `microcosm.frame` run every node, so binding them by rule would re-key every node on any edit to them. Only some value-computing modules outside `microcosm.build` are bound, by hand: `microcosm.fit`'s categorical, donor and held-out fitters in `spi_employment_shell` and `aps_occupation`, and the engine adapter in the stages that call the engine. These are still unbound: + - `microcosm.fit.qrf` and `fit.model`, which 11 stages fit through, those two among them; + - `microcosm.frame.rules` and `frame.materialize`; + - the engine adapter's helpers, in stages that call no engine; + - `microcosm.graph.randomness`, which `regional_home_values` draws through; + - `microcosm.calibrate.geography_constants`. +- **What is bound.** `national_frame` joins the modules every spine stage hashes. The stage helper table gains 231 bindings of 54 modules in 49 stages. Among them: + - in the CGT and high-value stages: `property_localisation`, `cgt_structure`, `spi_support`'s exact-total helpers, `cgt_imputation`, `cgt_asset_type` and `advani_summers`; + - `employer_pension_contributions` in `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`; + - the tables the narrower rule finds: `tenure_constants` in `was_wealth`, `was_lisa`, `high_value_home_support` and `regional_home_values`; `council_tax_constants` in `frs_spine`; `rowwise_geography` in `lcfs_consumption`; `etb_vat` in `etb_services`; `cgt_imputation` in `cgt_incidence_clone`; `frs_annual_income` and `hmrc_source_contract` in `hmrc_spi_income_spine`; + - `household_income` in `was_wealth` and `was_lisa`, for its predictor names. + + Two groups recur: the vendored-fact reader with the chronicle feed and its schema epochs (20 stages), and `frs_spine`'s pinned tab reader with `trace` (14 stages). The older import tests then ask for more, because the new helpers import these modules: + - `stochastic_assignment` is bound into 8 more stages, and the SPI support channel's exemption goes; + - the copy-id table is bound into `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`, and 15 stages join the copy-id test's name-only list; + - `frs_spine`, `spi_support_channel`, `spi_income_band_donors`, `uc_deduction_attributes` and `etb_services` join the counting stages. + + The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input`, `gates` and `spi_support`, whose replacement strata the sample reads. +- **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. It declares both modules through `graph_implementation_dependencies`, which the root node hashes. Its behaviour is unchanged. +- **Other dense kernels.** The rule is not yet applied to the UK's other graph kernels. Run over them, it finds unhashed modules in the identity node (`household_lineage`, `source_lineage`), the property value and band nodes (`tenure_constants`, `source_lineage`), the private rent node (`source_lineage`, `receipt_counts`, `national_frame`) and the BRMA and RFDR nodes (`receipt_counts`). The calibration and size kernels miss `national_frame`, and the size search and refit also miss the doctrine, local target and target weight modules. The rule finds more in the gate, target, national and export kernels. Binding them is left open. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler at every rung. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. diff --git a/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md b/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md index 933619afbd..99cef6fb89 100644 --- a/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md +++ b/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md @@ -1,5 +1,5 @@ The UK identity tool (`tools/verify_uk_identity_stability.py`) now scopes its receipts past the high-value home copies, and E5 checks the regional uprating on the values that stage left (#931). - **The copies.** The tool's stacking map gains `household_is_high_value_support_copy`, the flag of `high_value_home_support` (stage 26, microcosm#1240). The receipts that scope to an earlier stage now drop the copies and every CGT row made from them: E5's wealth block and E4's FRS-only frame. The copies' mass folds back by family. After the CGT folds, each copy's weight is added onto its root, so the root is back at its weight before the split. No uniform factor could do this, since only the top value stratum is divided. The fold decodes each copy at the copy-id table's `high_value_support` field. It refuses, by count, a copy without a root, a flag that disagrees with the stored index, an index that disagrees with the id, and a family whose decoded size disagrees with `high_value_support_copies`. Dropping the copies while a CGT layer stays in scope is refused. -- **E5's stored values.** `regional_home_values` (stage 27, microcosm#1112) rewrites `main_residence_value` and `property_wealth` after the uprating, so E5's fixed-point check of the uprating was comparing stage 27's values. The scoping now reverses that rewrite for every stage before stage 27. The value comes back from `main_residence_value_rank_basis`, which the stage keeps, and the wealth moves back by the same change. An artifact without the rank basis, such as a release, is read as before. -- **What moves.** Tool verdicts only. On the synthetic H2 oracle frame, E5's stored-column check fails with the parent's tool, as it does on origin/main, and passes now. The `was_wealth` scope drops the 115 high-value copies (of 404 rows) it kept. On that frame every high-value root is an SPI row, which E4 drops anyway, so E4 reads the same frame as before, and the E6, E7 and E9 receipts are identical. On a spine whose high-value roots include FRS households, E4's scope and weights change too. No stage, value, node key or generated file moves, since nothing hashes the tool. +- **E5's stored values.** `regional_home_values` (stage 27, microcosm#1112) rewrites `main_residence_value` and `property_wealth` after the uprating, so E5's fixed-point check of the uprating was comparing stage 27's values. The scoping now reverses that rewrite for every stage before stage 27. The value comes back from `main_residence_value_rank_basis`, which the stage keeps, and the wealth moves back by the same change. An artifact without the rank basis, such as a release, is read as before. The reversal refuses a roster in which a later stage declares either column, since it would then hide that stage's rewrite. +- **What moves.** Tool verdicts only. On the synthetic H2 oracle frame, E5's stored-column check failed with the tool before this change and passes now. On any spine built since `regional_home_values` joined the roster, the check no longer fails on that stage's rewrite. The `was_wealth` scope drops the 115 high-value copies (of 404 rows) it kept. On that frame every high-value root is an SPI row, which E4 drops anyway, so E4 reads the same frame as before, and the E6, E7 and E9 receipts are identical. On a spine whose high-value roots include FRS households, E4's scope and weights change too. No stage, value, node key or generated file moves, since nothing hashes the tool. diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index 92310d0600..19d8ec7a33 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -133,8 +133,11 @@ closed on 2026-09-28 by restoring the tool's behaviour on the driver. `::test_source_sampling_cannot_be_reapplied_as_pool_sampling`; the compact national sampler is not reachable from the graph's population node. (Note of 2026-11-08: microcosm#931 retired the compact national sampler, `sample_uk_national_frame`, which no build or tool - had called since 8adc836b9, together with that test. The second test and - `uk.full.sample`'s f100 branch, which calls no sampler, remain.) + had called since 8adc836b9, together with the first test. The second test is in + `test_uk_full_build_cli.py` and checks only that the config refuses a second sampling + fraction. `uk.full.sample`'s f100 branch is now pinned by + `test_uk_national_sampling.py::test_the_sample_node_calls_no_sampler_at_f100`, which runs + the node at fraction 1.0 with every sampler patched to fail and finds the spine unsampled.) 5. `candidate_engine_surface_reuses_one_resolver`: covered by `test_uk_full_measure.py::test_full_measure_reuses_one_resolver` (direct port). 6. `candidate_engine_surface_resolves_real_per_clone_blocks`: covered by diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 992f4db5b7..937702bd38 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -79,6 +79,8 @@ donor_uprating, employer_pension_contributions, energy_pricing, + etb_vat, + frs_annual_income, frs_brma, frs_disability, frs_education, @@ -93,6 +95,7 @@ hmrc_isa_statistics, hmrc_property_rental, hmrc_replay, + hmrc_source_contract, hmrc_uprating, household_income, household_lineage, @@ -114,6 +117,7 @@ stage_health, support_clip, take_up_contract, + tenure_constants, terminal_gates, uc_capital_coherence, uc_deduction_attributes, @@ -208,10 +212,15 @@ ) # Imported modules are not traversed by ``source_hash``, so each stage binds -# every module holding code its transform computes through, beyond its own -# module (microcosm#931). test_uk_graph derives that set: it follows what the -# transform names into the module defining it, to a fixed point, and needs no -# binding for a check, a column name, a record or a constant of strings. +# every module in microcosm.build holding code its transform computes +# through, beyond its own module (microcosm#931). test_uk_graph derives that +# set: it follows what the transform names into the module defining it, to a +# fixed point, and needs no binding for a listed refusal-only check. Where it +# stops at a single string, a column-name function or a record, the module is +# hashed by the stage or a stage upstream of it, or the name is listed with +# its reason; a collection of strings, like a number, is a value. Modules in +# the other workspace packages are bound by hand, and test_uk_graph names +# the ones that are not yet. # Every stage whose module or bound helpers import the shared keyed-draw # helpers (``microcosm.build.stochastic_assignment``), or import another # module's function that draws through them, binds them as well, so an edit @@ -295,13 +304,16 @@ # vendored facts and HMRC tables; the stage's parameters bound the band # donors and the coverage floor, and its result carries the fits' weight # kinds, gate outcomes and the HMRC replay report (microcosm#931). It - # binds the copy-id table as the leaves do. + # binds the copy-id table as the leaves do. The imputation refreshes the + # annual income descriptors on frs_annual_income's share columns, and the + # evidence shares iterate HMRC's distributional inputs. "hmrc_spi_income_spine": ( spi_income, spi_rank_transport, copy_id_table, employer_pension_contributions, *_FEED_FACT_MODULES, + frs_annual_income, frs_disability, frs_release, frs_spine, @@ -309,6 +321,7 @@ gates, hmrc_income, hmrc_replay, + hmrc_source_contract, hmrc_uprating, pension_coherence, qrf_gates, @@ -324,12 +337,14 @@ # high-value split weighs its copies through the CGT split's exact-total # correction (microcosm#931). The executor reads each id-offset stage's # declared radix from the table too. The root reads its tabs through - # trace's digest, ages people by birth month, and caps pension - # contributions with their receipt (microcosm#931). + # trace's digest, ages people by birth month, caps pension contributions + # with their receipt, and codes council tax bands in the shared band order + # (microcosm#931). "frs_spine": ( uc_relationships, copy_id_table, birth_month, + council_tax_constants, pension_coherence, receipt_counts, trace, @@ -356,13 +371,14 @@ stochastic_assignment, ), # The exact-total correction the CGT split hands the high-value split is - # spi_support's, and the support stratum is property_localisation's - # (microcosm#931). + # spi_support's, and the support stratum is property_localisation's, whose + # owners are the tenure map's (microcosm#931). "high_value_home_support": ( cgt_support, copy_id_table, property_localisation, spi_support, + tenure_constants, ), # The CGT stages read HMRC's CGT tables as vendored facts and the # Advani-Summers tail from advani_summers, take the CGT policy parameters @@ -370,7 +386,8 @@ # term), and weigh their copies through spi_support's exact-total # correction; the support split also takes the adult age and oldest-adult # rule from cgt_structure, and the residential split its logistic from - # cgt_asset_type (microcosm#931). + # cgt_asset_type; the incidence clone sums cgt_imputation's income proxy + # components into the income behind its propensity (microcosm#931). "cgt_support_split": ( copy_id_table, *_FEED_FACT_MODULES, @@ -386,6 +403,7 @@ rowwise_geography, copy_id_table, advani_summers, + cgt_imputation, spi_support, stochastic_assignment, ), @@ -429,14 +447,17 @@ # The donor stages read their donor tabs through frs_spine, record each # fit's weight kind, gate the fits and clip donor support to the donor's # range (microcosm#931); the WAS recipients' predictors map benefit units to - # households and count family roles. + # households, count family roles, group tenure by the shared tenure map and + # name the HBAI income predictor. "was_wealth": ( *_ENGINE_MODULES, *_PINNED_TAB_MODULES, frs_brma, gates, + household_income, qrf_gates, support_clip, + tenure_constants, uc_relationships, stochastic_assignment, ), @@ -451,7 +472,9 @@ *_FEED_FACT_MODULES, frs_brma, gates, + household_income, support_clip, + tenure_constants, uc_relationships, stochastic_assignment, ), @@ -472,8 +495,12 @@ stochastic_assignment, ), # The regional values rank as the dense node does and read its knots - # (microcosm#1112). - "regional_home_values": (property_localisation, rowwise_geography), + # (microcosm#1112), over the tenure map's owners (microcosm#931). + "regional_home_values": ( + property_localisation, + rowwise_geography, + tenure_constants, + ), # The purchases read the vendored HMRC Stamp Tax rows through the feed-pinned # reader and mark SDLT's regions from the FRS region map (microcosm#1095). "property_purchases": ( @@ -500,7 +527,8 @@ # through donor_uprating on vendored indices, the recipients' household # income and family roles are predictors, and bus fares are priced over # calendar and fiscal-year windows; the LCFS stage also rakes its energy, - # assigns car attributes and tops up gambling (microcosm#931). + # assigns car attributes, tops up gambling and reads the DfT boardings of + # each recipient's region through the FRS region map (microcosm#931). "lcfs_consumption": ( *_ENGINE_MODULES, energy_pricing, @@ -515,6 +543,7 @@ nts_car_attributes, qrf_gates, raking, + rowwise_geography, support_clip, stochastic_assignment, ), @@ -529,6 +558,8 @@ uc_relationships, stochastic_assignment, ), + # The services stage counts its recipients' family roles as the VAT stage + # does, from etb_vat's role table (microcosm#931). "etb_services": ( *_ENGINE_MODULES, *_PINNED_TAB_MODULES, @@ -537,6 +568,7 @@ bus_support_pricing, donor_uprating, energy_pricing, + etb_vat, gates, ledger_targets, qrf_gates, @@ -753,6 +785,7 @@ "nts_bus_travel", "lcfs_consumption", "etb_vat", + "etb_services", "uc_reporter_redraw", "spi_benefit_coherence", "uc_capital_coherence", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index 1ec7fffc04..b40855fa2a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -59,6 +59,7 @@ rfdr_area_layer, rowwise_dataset, rowwise_geography, + spi_support, ) from .geography_ladder import ( UK_GEOGRAPHY_LADDER_COLUMNS, @@ -203,10 +204,11 @@ def implementation_hash(self) -> str: # The dense population nodes also compute through what these call: # the sample's household draw and mass normalisation (frame_sampling), - # the clones' minted ids and copied tables (rowwise_geography), the - # national frame they rebuild, the local authority names the geography - # resolves, and the gate's results (microcosm#931); test_uk_graph - # derives the set from what the kernels name. + # its strata columns (spi_support's replacement strata), the clones' + # minted ids and copied tables (rowwise_geography), the national frame + # they rebuild, the local authority names the geography resolves, and + # the gate's results (microcosm#931); test_uk_graph derives the set + # from what the kernels name. return source_hash( type(self), geography_ladder, @@ -218,6 +220,7 @@ def implementation_hash(self) -> str: local_authority_input, national_frame, rowwise_geography, + spi_support, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py index a898110e10..332ab54b39 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py @@ -244,7 +244,10 @@ def split_high_value_support_households( records ``k`` in ``high_value_support_copy_index`` and carries a full copy of its person and benefit-unit rows. The exact-total correction lands on the heaviest unselected incumbent so every copy weight stays bitwise - ``w / copies``. Refuses a frame this stage has already split. + ``w / copies``, unless that household cannot carry it: then spi_support's + shared fallback places it, and that row can be a root or copy + (:func:`microcosm.build.uk_runtime.cgt_support._exact_support_weights`). + Refuses a frame this stage has already split. """ if isinstance(copies, bool) or copies < 2: raise ValueError("High-value support copies must be an integer of at least 2.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py index a90978b607..825b58fa7e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py @@ -17,16 +17,21 @@ A spine without lineage columns is sampled household by household, stratified by region. The module also holds the rung tokens and the default -sampling seed that the UK builds share. +sampling seed that the UK builds share, and the spine build's root sampler: +a spine built below f100 samples its FRS root household by household +(:func:`sample_uk_spine_root`) through :class:`UKSampledSpineRootTransform`, +which the root node hashes with this module and the shared core. """ from __future__ import annotations +import sys from collections.abc import Mapping import numpy as np import pandas as pd +from microcosm.build import frame_sampling from microcosm.build.frame_sampling import ( normalize_sampled_household_mass, sample_frame_households, @@ -43,7 +48,9 @@ __all__ = [ "UK_SAMPLE_RUNG_TOKENS", "UK_SAMPLE_SEED_DEFAULT", + "UKSampledSpineRootTransform", "sample_uk_spine_frame", + "sample_uk_spine_root", "uk_spine_source_family_units", ] @@ -56,8 +63,7 @@ } #: Default survey-sampling seed, matching the US stacked pipeline's. Distinct -#: from the build ``--seed`` (SPI replacement draw, donor bootstrap, QRF) so -#: dev-scale seed sweeps vary one draw at a time. +#: from the build ``--seed`` so dev-scale seed sweeps vary one draw at a time. UK_SAMPLE_SEED_DEFAULT = 578 ( @@ -278,3 +284,117 @@ def sample_uk_spine_frame( "strata_count": int(len(np.unique(strata))), "receipt": dict(core_receipt), } + + +def sample_uk_spine_root( + frame: Frame, + *, + fraction: float, + seed: int, +) -> tuple[Frame, dict[str, object] | None]: + """Sample the spine build's FRS root at a rung below f100. + + Households are drawn by the shared core and their mass is renormalized + to the root's household total. At f100 the frame is returned as it is, + with no receipt. + """ + + if fraction == 1.0: + return frame, None + household_weight = frame.weights_for("household") + pre_households = int(len(frame.table("household"))) + sampled, receipt = sample_frame_households( + frame, + fraction=fraction, + seed=seed, + source_name="UK FRS spine", + ) + normalized, factor = normalize_sampled_household_mass( + sampled, + target_mass=float(household_weight.total), + source_name="UK FRS spine", + ) + return normalized, { + "fraction": float(fraction), + "seed": int(seed), + "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], + "pre_household_count": pre_households, + "post_household_count": int(len(normalized.table("household"))), + "normalization_factor": float(factor), + "receipt": dict(receipt), + } + + +class UKSampledSpineRootTransform: + """CREATE-stage adapter applying the declared sampling rung at ingest. + + The spine build wraps the FRS root's transform in it. Its sampling code + lives here and in the shared core, not in the root stage's module, so it + declares both for the root node to hash (microcosm#931). + """ + + def __init__( + self, + transform, + *, + fraction: float, + seed: int, + ) -> None: + self.transform = transform + self.fraction = fraction + self.seed = seed + self.sampling: dict[str, object] | None = None + + def graph_implementation_dependencies(self) -> tuple[object, ...]: + """The modules the root node hashes for this adapter's sampling.""" + + return (sys.modules[__name__], frame_sampling) + + def _sample(self, assembled): + sampled, self.sampling = sample_uk_spine_root( + assembled, + fraction=self.fraction, + seed=self.seed, + ) + # Graph populations use row positions as their internal alignment + # index. Frame sampling preserves source DataFrame indexes by design, + # so normalize those indexes at this adapter boundary. + tables = { + entity: sampled.table(entity).reset_index(drop=True) + for entity in sampled.entities + } + tables.update( + {name: sampled.link(name).reset_index(drop=True) for name in sampled.links} + ) + return Frame( + tables, + sampled.schema, + { + entity: sampled.weights_for(entity) + for entity in sampled.weighted_entities + }, + sampled.strata.reset_index(drop=True), + mass_log=sampled.mass_log, + metadata=sampled.metadata, + ) + + def effective_fraction(self) -> float: + """Return the configured input sampling fraction.""" + + return float(self.fraction) + + def __call__(self, frame): + return self._sample(self.transform(frame)) + + def run_with_sources(self, frame, sources): + runner = getattr(self.transform, "run_with_sources", None) + assembled = ( + runner(frame, sources) if callable(runner) else self.transform(frame) + ) + return self._sample(assembled) + + def checkpoint_metadata(self) -> dict[str, object]: + hook = getattr(self.transform, "checkpoint_metadata", None) + if not callable(hook): + raise RuntimeError("FRS root transform exposes no checkpoint metadata.") + return dict(hook()) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index ef1edfbce5..38db1087a7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -21,10 +21,6 @@ GatesManifest, load_country_spec, ) -from microcosm.build.frame_sampling import ( - normalize_sampled_household_mass, - sample_frame_households, -) from microcosm.build.gate_battery import ( BlockingMode, EvidenceContext, @@ -161,6 +157,8 @@ from microcosm.build.uk_runtime.national_sampling import ( UK_SAMPLE_RUNG_TOKENS, UK_SAMPLE_SEED_DEFAULT, + UKSampledSpineRootTransform, + sample_uk_spine_root, ) from microcosm.build.uk_runtime.nts_bus_travel import UKNTSBusTravelStageTransform from microcosm.build.uk_runtime.orrery_contract import ( @@ -230,7 +228,6 @@ from microcosm.build.uk_runtime.uc_take_up import UKUCTakeUpStageTransform from microcosm.build.uk_runtime.was_lisa import UKWASLISAStageTransform from microcosm.build.uk_runtime.was_wealth import UKWASWealthStageTransform -from microcosm.frame import Frame from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine from microcosm.graph import ( ContentStore, @@ -1180,103 +1177,6 @@ def _record_attempt( ) -def _sample_spine_frame( - frame, - *, - fraction: float, - seed: int, -) -> tuple[object, dict[str, object] | None]: - if fraction == 1.0: - return frame, None - household_weight = frame.weights_for("household") - pre_households = int(len(frame.table("household"))) - sampled, receipt = sample_frame_households( - frame, - fraction=fraction, - seed=seed, - source_name="UK FRS spine", - ) - normalized, factor = normalize_sampled_household_mass( - sampled, - target_mass=float(household_weight.total), - source_name="UK FRS spine", - ) - return normalized, { - "fraction": float(fraction), - "seed": int(seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], - "pre_household_count": pre_households, - "post_household_count": int(len(normalized.table("household"))), - "normalization_factor": float(factor), - "receipt": dict(receipt), - } - - -class _SampledGraphRootTransform: - """CREATE-stage adapter applying the declared sampling rung at ingest.""" - - def __init__( - self, - transform, - *, - fraction: float, - seed: int, - ) -> None: - self.transform = transform - self.fraction = fraction - self.seed = seed - self.sampling: dict[str, object] | None = None - - def _sample(self, assembled): - sampled, self.sampling = _sample_spine_frame( - assembled, - fraction=self.fraction, - seed=self.seed, - ) - # Graph populations use row positions as their internal alignment - # index. Frame sampling preserves source DataFrame indexes by design, - # so normalize those indexes at this adapter boundary. - tables = { - entity: sampled.table(entity).reset_index(drop=True) - for entity in sampled.entities - } - tables.update( - {name: sampled.link(name).reset_index(drop=True) for name in sampled.links} - ) - return Frame( - tables, - sampled.schema, - { - entity: sampled.weights_for(entity) - for entity in sampled.weighted_entities - }, - sampled.strata.reset_index(drop=True), - mass_log=sampled.mass_log, - metadata=sampled.metadata, - ) - - def effective_fraction(self) -> float: - """Return the configured input sampling fraction.""" - - return float(self.fraction) - - def __call__(self, frame): - return self._sample(self.transform(frame)) - - def run_with_sources(self, frame, sources): - runner = getattr(self.transform, "run_with_sources", None) - assembled = ( - runner(frame, sources) if callable(runner) else self.transform(frame) - ) - return self._sample(assembled) - - def checkpoint_metadata(self) -> dict[str, object]: - hook = getattr(self.transform, "checkpoint_metadata", None) - if not callable(hook): - raise RuntimeError("FRS root transform exposes no checkpoint metadata.") - return dict(hook()) - - def _stage_observer( staging_bundle: StagingRunBundleWriterV2 | None, emitter: LocalTelemetryEmitter | None, @@ -1379,7 +1279,7 @@ def _run_plan_with_spine_sampling( spine_frame, spine_records = StagePlan(plan.stages[:1]).run( uk_frs_spine_seed_frame() ) - spine_frame, sampling = _sample_spine_frame( + spine_frame, sampling = sample_uk_spine_root( spine_frame, fraction=sample_fraction, seed=sample_seed, @@ -2065,7 +1965,7 @@ def prepare_uk_spine_execution( stage=fixture_by_name["frs_hmrc_spine_leaves"], sampled_rung=True, ) - sampled_root = _SampledGraphRootTransform( + sampled_root = UKSampledSpineRootTransform( implementations["frs_spine"], fraction=args.sample_fraction, seed=args.sample_seed, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 7c95aad74f..b6364104a6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -1286,9 +1286,10 @@ def _leaves(value): # (9,000) is the first candidate for the exact-total correction (on clone 0, # the first heaviest). At seven (and 13) clones its clone 0 has no # representable correction, so the shared fallback puts it on the last copy -# of the 7,620 family's last clone, a few ulps off w / n and within the -# weights' nesting tolerance and the gate's; at one, three, 11 and 15 clones -# no correction is needed. +# of the 7,620 family's last clone. That copy is then off w / n by about an +# ulp of the frame's total weight (thousands of ulps of its own, a relative +# error below 1e-12), within the weights' nesting tolerance and the gate's; +# at one, three, 11 and 15 clones no correction is needed. NESTING_WEIGHTS = [ 900.0, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index a3d3d2c7a4..ff1154876c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -3841,8 +3841,11 @@ def test_driver_records_sampled_spine_sidecar( monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + # The root's rung sampler lives in national_sampling (microcosm#931). + from microcosm.build.uk_runtime import national_sampling + monkeypatch.setattr( - tool, + national_sampling, "sample_frame_households", lambda frame, **kwargs: ( frame, @@ -3850,7 +3853,7 @@ def test_driver_records_sampled_spine_sidecar( ), ) monkeypatch.setattr( - tool, + national_sampling, "normalize_sampled_household_mass", lambda frame, **kwargs: (frame, 1.0), ) @@ -4400,7 +4403,10 @@ def test_driver_sampled_named_edge_aborts_with_receipt( def _raise_named_edge(*args, **kwargs): raise ValueError(tool._RUNG_NAMED_EDGE_SIGNATURE) - monkeypatch.setattr(tool, "sample_frame_households", _raise_named_edge) + # The root's rung sampler lives in national_sampling (microcosm#931). + from microcosm.build.uk_runtime import national_sampling + + monkeypatch.setattr(national_sampling, "sample_frame_households", _raise_named_edge) assert ( tool.main( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 87f430295f..84f18252dc 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -452,7 +452,7 @@ def changed_helper_bytes(path: Path) -> bytes: #: bind them: the stage's transform, the drawing names its module imports, and #: why the transform never calls them. None at present: the support channel, #: the one stage listed before, binds frs_take_up, which imports the helpers, -#: and so binds them too (microcosm#931). +#: so it must list them too, and does (microcosm#931). _STOCHASTIC_ASSIGNMENT_UNBOUND: dict[str, tuple[str, frozenset[str], str]] = {} @@ -884,9 +884,16 @@ def changed_module_bytes(path): #: The package the rule follows: microcosm.build holds every UK stage and its #: helpers. The workspace's other packages stay outside it. microcosm.graph #: and microcosm.frame are the executor and the Frame that every node runs -#: on, so binding them would re-key every node of every graph on any edit to -#: them; microcosm.fit's fitters, shared by every country, are bound by hand -#: (the employment shells above) and not derived here. +#: on, so binding them by rule would re-key every node of every graph on any +#: edit to them. Value-computing code out there is bound only by hand: +#: microcosm.fit's categorical, donor and held-out fitters in +#: spi_employment_shell and aps_occupation, and the engine adapter in the +#: stages that call the engine. The rest is not bound yet: microcosm.fit.qrf +#: and fit.model, which the QRF donor stages and those two fit through; +#: microcosm.frame.rules and frame.materialize; the engine adapter's helpers +#: in stages that call no engine; microcosm.graph.randomness, which +#: regional_home_values draws through; and +#: microcosm.calibrate.geography_constants. _HELPER_SCOPE = "microcosm.build." #: Where the rule stops. What a stage reads from these is its own #: declaration, which its node key already hashes as the stage contract: its @@ -897,8 +904,9 @@ def changed_module_bytes(path): "microcosm.build.spec_engine", ) #: Checks that read values only to decide whether to refuse, each with why it -#: cannot change what its caller computes. The rule does not follow them; the -#: test checks that each returns nothing and writes nothing. +#: cannot change what its caller computes. The rule does not follow them, and +#: a definition that names nothing else of value needs no binding for them; +#: test_the_refusal_only_helpers_return_and_write_nothing checks each one. _REFUSAL_ONLY_HELPERS: dict[tuple[str, str], str] = { ( "microcosm.build.uk_runtime.graph_evidence", @@ -907,24 +915,76 @@ def changed_module_bytes(path): "refuses a node unless the spine gates admitted its input; the dense " "sample node calls it before it computes anything" ), + ("microcosm.build.uk_runtime.cgt_structure", "_assert_closed_world_operations"): ( + "compares a stage's declared operations with the reviewed mapping, in " + "order and in full, and refuses on any difference; the stages' " + "parameter checks call it" + ), + ("microcosm.build.uk_runtime.spi_support", "_require_entity"): ( + "returns its argument unchanged and refuses anything but person, " + "benunit or household; the support channel's column names call it" + ), +} +#: Names a stage reads from a module that neither its node nor any node +#: upstream of it hashes, each with why an edit to them cannot leave the +#: stage's output stale. A name hashed upstream needs no entry: node keys +#: chain on their predecessors' keys, so an edit there re-keys the stage too. +#: Only a single string, a function returning one column name, or a record +#: can be listed (``_helper_kind``); the tests hold the table to exactly what +#: they find, so a new one is bound or listed here with its reason. +_STAGE_UNBOUND_NAMES: dict[tuple[str, str], str] = { + ("microcosm.build.uk_runtime.rowwise_dataset", "ladder_clone_index_column"): ( + "names the geographic clone index columns, which only the dense " + "expand writes; no spine stage writes them (checked), so a spine " + "frame never carries them and their name moves no spine value" + ), } -#: Method names that change their receiver in place. +#: The same for the dense population kernels, which run outside the spine +#: graph. +_DENSE_UNBOUND_NAMES: dict[tuple[str, str], str] = { + ("microcosm.build.uk_runtime.geography_sources", name): ( + "the LAD23 names lookup's pinned URL and digest: local_authority_input " + "refuses an input whose declared source differs from them, and reads " + "nothing through them" + ) + for name in ("LAD23_NAMES_SHA256", "LAD23_NAMES_URL") +} +#: Method and function names that change their receiver or an argument in +#: place. _IN_PLACE_CALLS = frozenset( { + "__delitem__", "__setattr__", "__setitem__", "add", "append", + "appendleft", + "at", "clear", + "copyto", + "difference_update", "discard", "extend", + "extendleft", "fill", + "fill_diagonal", "insert", + "intersection_update", + "itemset", + "place", "pop", "popitem", + "put", + "putmask", "remove", + "resize", + "reverse", + "rotate", "setdefault", + "setflags", + "shuffle", "sort", + "symmetric_difference_update", "update", } ) @@ -1066,8 +1126,11 @@ def _named(module: str, node, scope: str) -> set[tuple[str, str]]: def _writes_nothing(node) -> bool: - """Whether a function returns nothing but None or an argument it never - rebinds, and writes through no subscript, attribute or in-place call.""" + """Whether a function can only refuse: it returns nothing but None, or its + sole parameter unrebound, and writes nothing: no store through a + subscript or attribute, no augmented assignment, no ``out=`` or + ``inplace=`` argument, and no call by a name that changes its receiver or + an argument in place.""" import ast @@ -1086,17 +1149,20 @@ def _writes_nothing(node) -> bool: for sub in ast.walk(node) if isinstance(sub, ast.Name) and isinstance(sub.ctx, ast.Store | ast.Del) } + returnable = parameters - rebound if len(parameters) == 1 else set() for sub in ast.walk(node): if isinstance(sub, ast.Return) and sub.value is not None: value = sub.value if not ( (isinstance(value, ast.Constant) and value.value is None) - or (isinstance(value, ast.Name) and value.id in parameters - rebound) + or (isinstance(value, ast.Name) and value.id in returnable) ): return False - elif isinstance(sub, ast.Yield | ast.YieldFrom | ast.Global | ast.Nonlocal): + elif isinstance( + sub, ast.AugAssign | ast.Yield | ast.YieldFrom | ast.Global | ast.Nonlocal + ): return False - elif isinstance(sub, ast.Assign | ast.AugAssign | ast.AnnAssign | ast.Delete): + elif isinstance(sub, ast.Assign | ast.AnnAssign | ast.Delete): targets = ( sub.targets if isinstance(sub, ast.Assign | ast.Delete) @@ -1108,13 +1174,14 @@ def _writes_nothing(node) -> bool: for part in ast.walk(target) ): return False - elif isinstance(sub, ast.keyword) and sub.arg == "inplace": + elif isinstance(sub, ast.keyword) and sub.arg in {"inplace", "out"}: return False elif isinstance(sub, ast.Call): function = sub.func if isinstance(function, ast.Attribute) and function.attr in _IN_PLACE_CALLS: return False if isinstance(function, ast.Name) and function.id in { + *_IN_PLACE_CALLS, "delattr", "eval", "exec", @@ -1147,50 +1214,24 @@ def _string_expression(node) -> bool: return True -def _only_strings(node) -> bool: - """Whether an expression is built of string literals, None, and names, - in literal collections: a name, a code, or a collection of them.""" - - import ast - - collections = {"MappingProxyType", "dict", "frozenset", "list", "set", "tuple"} - for sub in ast.walk(node): - if isinstance(sub, ast.Constant): - if not (sub.value is None or isinstance(sub.value, str)): - return False - elif isinstance(sub, ast.Call): - if not (isinstance(sub.func, ast.Name) and sub.func.id in collections): - return False - elif not isinstance( - sub, - ast.Name - | ast.Attribute - | ast.Tuple - | ast.List - | ast.Set - | ast.Dict - | ast.JoinedStr - | ast.FormattedValue - | ast.expr_context, - ): - return False - return True - - _HELPER_KINDS: dict[tuple[str, str, str], str | None] = {} def _helper_kind(module: str, name: str, scope: str) -> str: """How a top-level definition can bear on a value. - ``value`` unless it is provably one of: a ``check`` (a function that - writes nothing and names only definitions of these kinds, so it can only - refuse); a ``column name`` (a function returning one string built from - its arguments); a ``record`` (a class of undefaulted fields, no methods); - ``names`` (a constant of strings: a column name, a code, or a collection - of them). A numeric constant is a value. A cycle reads as ``value``. + ``value`` unless it is a listed refusal-only ``check`` + (``_REFUSAL_ONLY_HELPERS``) or provably one of: a ``name`` (one string: a + literal, an f-string or concatenation of names, or another name of these + kinds); a ``column name`` (a function returning one such string built + from its arguments); a ``record`` (a class of undefaulted fields whose + types are of these kinds, no methods). A collection of strings is a + value, since it selects rows, columns or codes, as is a number. A cycle + reads as ``value``. """ + if (module, name) in _REFUSAL_ONLY_HELPERS: + return "check" key = (module, name, scope) if key in _HELPER_KINDS: return _HELPER_KINDS[key] or "value" @@ -1213,12 +1254,16 @@ def _classify_helper(module: str, name: str, scope: str) -> str: node = _module_definitions(module, scope).definitions[name] if isinstance(node, ast.Assign | ast.AnnAssign): if node.value is None or ( - _only_strings(node.value) + ( + isinstance(node.value, ast.Name | ast.Attribute) + or _string_expression(node.value) + ) and _names_only_harmless(module, node.value, scope) ): - return "names" + return "name" return "value" if isinstance(node, ast.ClassDef): + fields = [item for item in node.body if isinstance(item, ast.AnnAssign)] fields_only = all( (isinstance(item, ast.Expr) and isinstance(item.value, ast.Constant)) or isinstance(item, ast.Pass) @@ -1227,7 +1272,12 @@ def _classify_helper(module: str, name: str, scope: str) -> str: ) if fields_only and all( _names_only_harmless(module, item, scope) - for item in (*node.bases, *node.keywords, *node.decorator_list) + for item in ( + *node.bases, + *node.keywords, + *node.decorator_list, + *(field.annotation for field in fields), + ) ): return "record" return "value" @@ -1245,20 +1295,23 @@ def _classify_helper(module: str, name: str, scope: str) -> str: and _names_only_harmless(module, body[0].value, scope) ): return "column name" - if _writes_nothing(node) and _names_only_harmless(module, node, scope): - return "check" return "value" -def _helper_reach( +def _helper_walk( root: tuple[str, str], scope: str = _HELPER_SCOPE -) -> dict[str, frozenset[str]]: - """Every module holding a definition that can change what ``root`` - computes, with those definitions: ``root`` itself, and each ``value`` - definition it names, and they name, to a fixed point. The declaration - modules and the refusal-only checks are not followed.""" +) -> tuple[dict[str, frozenset[str]], frozenset[tuple[str, str]]]: + """What ``root`` can compute through, and where the rule stops. + + The first part maps every module holding a definition that can change + what ``root`` computes to those definitions: ``root`` itself, and each + ``value`` definition it names, and they name, to a fixed point. The second + is every ``name``, ``column name`` or ``record`` those definitions name, + which the rule does not follow. The declaration modules and the listed + refusal-only checks are neither followed nor returned.""" reached: dict[str, set[str]] = {} + stops: set[tuple[str, str]] = set() pending, seen = [root], set() while pending: item = pending.pop() @@ -1266,14 +1319,30 @@ def _helper_reach( continue seen.add(item) module, name = item - if module.startswith(_DECLARATION_MODULES) or item in _REFUSAL_ONLY_HELPERS: - continue - if item != root and _helper_kind(module, name, scope) != "value": + if module.startswith(_DECLARATION_MODULES): continue + if item != root: + kind = _helper_kind(module, name, scope) + if kind != "value": + if kind != "check": + stops.add(item) + continue reached.setdefault(module, set()).add(name) node = _module_definitions(module, scope).definitions[name] pending.extend(_named(module, node, scope)) - return {module: frozenset(names) for module, names in reached.items()} + return ( + {module: frozenset(names) for module, names in reached.items()}, + frozenset(stops), + ) + + +def _helper_reach( + root: tuple[str, str], scope: str = _HELPER_SCOPE +) -> dict[str, frozenset[str]]: + """Every module holding a definition that can change what ``root`` + computes, with those definitions (``_helper_walk``).""" + + return _helper_walk(root, scope)[0] def _stage_transforms() -> dict[str, tuple[str, str]]: @@ -1318,6 +1387,27 @@ def _stage_transforms() -> dict[str, tuple[str, str]]: return transforms +@functools.cache +def _stage_walks() -> dict[str, tuple[str, frozenset[str], tuple]]: + """Each spine stage's node id, the modules its node hashes, and the + walk from its transform (``_helper_walk``).""" + + from microcosm.build.uk_runtime import graph_kernels + + graph = uk_spine_graph() + registry = uk_registry(graph=graph) + walks = {} + for stage, transform in sorted(_stage_transforms().items()): + assert transform[0] == graph_kernels._stage_module(stage).__name__, stage + # The root stage's node is the CREATE kernel. + node = graph.node("create_uk_frs" if stage == "frs_spine" else stage) + hashed = frozenset( + hashed_module_names(registry.get(node.kernel), graph_kernels) + ) + walks[stage] = (node.id, hashed, _helper_walk(transform)) + return walks + + def test_every_stage_hashes_each_helper_its_transform_computes_through() -> None: """A stage's hash names every module holding code its transform can run to compute a value, so an edit to any of them re-keys the stage instead @@ -1326,33 +1416,78 @@ def test_every_stage_hashes_each_helper_its_transform_computes_through() -> None The rule starts at the transform graph_kernels builds for the stage and follows every definition it names, annotations included (an injected contract or table is called through them), into the module defining it. - It needs no binding, and does not follow, a definition that cannot change - a value: a check, a column name, a record, a constant of strings (see - ``_helper_kind``), or a stage declaration. A numeric constant needs its - module bound. The rule starts at the transform, not at the kernel that - runs it: the kernel's code, in graph_kernels, which every stage hashes, - builds every stage's transform for the hermetic registry, so following it - would bind every helper into every stage. + It does not follow a listed refusal-only check or a stage declaration, + and stops at a single string, a column-name function or a record (see + ``_helper_kind``), whose module the next test accounts for. Anything + else is a value, a collection of strings and a number included, and + needs its module bound. The rule starts at the transform, not at the + kernel that runs it: the kernel's code, in graph_kernels, which every + stage hashes, builds every stage's transform for the hermetic registry, + so following it would bind every helper into every stage. What the rule cannot see: a call through an object whose class the code never names, a name looked up by string, and data files read by path. """ from microcosm.build.uk_runtime import graph_kernels + walks = _stage_walks() + assert set(walks) == set(graph_kernels._STAGE_MODULES) + unhashed = { + stage: sorted(set(reach) - hashed) + for stage, (_, hashed, (reach, _)) in walks.items() + if set(reach) - hashed + } + assert not unhashed, unhashed + + +def test_every_name_a_stage_reads_is_hashed_with_it_upstream_or_listed() -> None: + """Where the rule stops at a name, the name's module is hashed by the + stage's node or by a node upstream of it, whose key the stage's key + chains on, or the name is listed in ``_STAGE_UNBOUND_NAMES`` with its + reason; the table lists nothing else (microcosm#931).""" + from microcosm.build.uk_runtime import graph_kernels, rowwise_dataset + from microcosm.graph.decl import compile_graph + graph = uk_spine_graph() + compiled = compile_graph(graph) + walks = _stage_walks() + hashed_by_node = {node_id: hashed for node_id, hashed, _ in walks.values()} registry = uk_registry(graph=graph) - transforms = _stage_transforms() - assert set(transforms) == set(graph_kernels._STAGE_MODULES) - unhashed = {} - for stage, transform in sorted(transforms.items()): - assert transform[0] == graph_kernels._stage_module(stage).__name__, stage - # The root stage's node is the CREATE kernel. - node = graph.node("create_uk_frs" if stage == "frs_spine" else stage) - hashed = hashed_module_names(registry.get(node.kernel), graph_kernels) - missing = sorted(set(_helper_reach(transform)) - hashed) - if missing: - unhashed[stage] = missing - assert not unhashed, unhashed + for node in graph.nodes: + if node.id not in hashed_by_node: + hashed_by_node[node.id] = frozenset( + hashed_module_names(registry.get(node.kernel), graph_kernels) + ) + + def upstream(node_id: str) -> set[str]: + seen, pending = set(), list(compiled.predecessors[node_id]) + while pending: + item = pending.pop() + if item not in seen: + seen.add(item) + pending.extend(compiled.predecessors[item]) + return seen + + unlisted: dict[tuple[str, str], list[str]] = {} + for stage, (node_id, hashed, (_, stops)) in walks.items(): + covered = set(hashed).union( + *(hashed_by_node[item] for item in upstream(node_id)) + ) + for module, name in stops: + if module not in covered: + unlisted.setdefault((module, name), []).append(stage) + assert set(unlisted) == set(_STAGE_UNBOUND_NAMES), sorted( + set(unlisted) ^ set(_STAGE_UNBOUND_NAMES) + ) + # The geographic clone index columns' reason: no spine node writes them. + written = { + (owned.entity, owned.column) for node in graph.nodes for owned in node.outputs + } + for entity in ("person", "benunit", "household"): + assert ( + entity, + rowwise_dataset.ladder_clone_index_column(entity), + ) not in written def test_the_refusal_only_helpers_return_and_write_nothing() -> None: @@ -1361,14 +1496,48 @@ def test_the_refusal_only_helpers_return_and_write_nothing() -> None: assert _writes_nothing(node), (module, name) +def test_the_refusal_check_sees_each_way_a_function_writes() -> None: + """Control for ``_writes_nothing``: a function that only refuses, or + returns its sole argument, passes; one that writes through any of these + shapes, or returns one of several arguments, does not.""" + import ast + + refusing = { + "raise": "def f(frame):\n if frame is None:\n raise ValueError('x')\n", + "identity": ( + "def f(entity):\n" + " if entity not in {'person'}:\n" + " raise ValueError('x')\n" + " return entity\n" + ), + } + writing = { + "copyto": "def f(values, mask):\n np.copyto(values, 0.0, where=mask)\n", + "out": "def f(values):\n np.clip(values, 0, 1, out=values)\n", + "add_at": "def f(w, roots, copies):\n np.add.at(w, roots, w[copies])\n", + "shuffle": "def f(rng, values):\n rng.shuffle(values)\n", + "reverse": "def f(items):\n items.reverse()\n", + "augmented": "def f(values):\n values *= 2.0\n", + "subscript": "def f(values):\n values[0] = 1.0\n", + "inplace": "def f(table):\n table.drop(columns=['a'], inplace=True)\n", + "pick": "def f(left, right):\n return left\n", + "computed": "def f(values):\n return values + 1\n", + } + for label, source in {**refusing, **writing}.items(): + function = ast.parse(source).body[0] + assert _writes_nothing(function) is (label in refusing), label + + def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( tmp_path, monkeypatch ) -> None: """Control: a helper called by name, through a re-export, by attribute on its module, through an alias or a computed constant, or held only in an - annotation is reached, as is a numeric constant; a check, a column name, - a record, a constant of strings, and a helper the root never names are - not.""" + annotation (a record's field type included) is reached, as are a numeric + constant, a collection of strings and a function that writes in place or + only refuses but is not listed; a listed check, a single string, a column + name, a record of plain fields, and a helper the root never names are + not, and the walk returns the names it stopped at.""" import uuid scratch = f"helper_rule_{uuid.uuid4().hex[:8]}" @@ -1386,10 +1555,28 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( "collaborator": ( "class Contract:\n def rate(self, key):\n return 0.5\n" ), + "fielded": ( + "from typing import NamedTuple\n\n" + "class Rate:\n def value(self):\n return 0.1\n\n" + "class Inputs(NamedTuple):\n rate: Rate\n" + ), "numeric": "FLOOR = 10\n", + "codes": ( + "CODES = frozenset({'A', 'B'})\n" + "OWNER = {'OWNED': 'owner', 'RENTED': 'renter'}\n" + ), + "inplace": ( + "import numpy as np\n\n" + "def fold(weights, roots, copies):\n" + " np.add.at(weights, roots, weights[copies])\n" + ), + "guard": ( + "def guard(frame):\n" + " if frame is None:\n" + " raise ValueError('no frame')\n" + ), "harmless": ( - "COLUMN = 'household_weight'\n" - "CODES = frozenset({'A', 'B'})\n\n" + "COLUMN = 'household_weight'\n\n" "def check(frame):\n" " if frame is None:\n" " raise ValueError('no frame')\n\n" @@ -1402,21 +1589,30 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( "stage": ( "from . import attribute\n" "from .alias import scale\n" + "from .codes import CODES, OWNER\n" "from .collaborator import Contract\n" "from .computed import TABLE\n" "from .direct import draw\n" - "from .harmless import CODES, COLUMN, Record, check, column\n" + "from .fielded import Inputs\n" + "from .guard import guard\n" + "from .harmless import COLUMN, Record, check, column\n" + "from .inplace import fold\n" "from .numeric import FLOOR\n" "from .reexport import total\n" "from .unused import never\n\n" "class Transform:\n" - " contract: Contract\n\n" + " contract: Contract\n" + " inputs: Inputs\n\n" " def __call__(self, frame):\n" " check(frame)\n" + " guard(frame)\n" + " fold(frame, [0], [1])\n" " record = Record(value=draw(1) + total([1]))\n" " rate = self.contract.rate(column('household'))\n" + " rate += self.inputs.rate.value()\n" + " owners = [OWNER[code] for code in CODES]\n" " return attribute.share(scale(rate)) + TABLE['a'] + FLOOR, (\n" - " record, COLUMN, CODES\n" + " record, COLUMN, owners\n" " )\n\n" "def helper():\n" " return never(1)\n" @@ -1425,7 +1621,10 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( for module, source in sources.items(): (package / f"{module}.py").write_text(source) prefix = f"microcosm.{scratch}." - reach = _helper_reach((f"{prefix}stage", "Transform"), scope=prefix) + monkeypatch.setitem( + _REFUSAL_ONLY_HELPERS, (f"{prefix}harmless", "check"), "control" + ) + reach, stops = _helper_walk((f"{prefix}stage", "Transform"), scope=prefix) assert {module.removeprefix(prefix): names for module, names in reach.items()} == { "stage": {"Transform"}, "direct": {"draw"}, @@ -1434,7 +1633,16 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( "alias": {"scale", "_scale"}, "computed": {"TABLE", "_build"}, "collaborator": {"Contract"}, + "fielded": {"Inputs", "Rate"}, "numeric": {"FLOOR"}, + "codes": {"CODES", "OWNER"}, + "inplace": {"fold"}, + "guard": {"guard"}, + } + assert {(module.removeprefix(prefix), name) for module, name in stops} == { + ("harmless", "COLUMN"), + ("harmless", "Record"), + ("harmless", "column"), } @@ -1442,9 +1650,11 @@ def test_the_dense_population_kernels_hash_each_helper_they_compute_through() -> """The same rule for the dense population kernels that share ``_PopulationKernel``'s hash (sample, normalize, expand, locations, geography mapping, pool and gate): every module holding code one of them - can run to compute a value is in the hash they share (microcosm#931). - The rule starts at each kernel's class; the spine gate admission check - the sample kernel calls is refusal-only and is not followed.""" + can run to compute a value is in the hash they share, and every name the + rule stops at outside it is listed in ``_DENSE_UNBOUND_NAMES`` with its + reason (microcosm#931). The rule starts at each kernel's class; the spine + gate admission check the sample kernel calls is refusal-only and is not + followed.""" import inspect from microcosm.build.uk_runtime import graph_population, rowwise_geography @@ -1464,16 +1674,147 @@ def test_the_dense_population_kernels_hash_each_helper_they_compute_through() -> "uk.full.geography_gate@1", } unhashed = {} + unlisted = set() for kernel in kernels: hashed = hashed_module_names(kernel.__new__(kernel), graph_population) # The dense clones are minted by rowwise_geography's helpers. assert rowwise_geography.__name__ in hashed - missing = sorted( - set(_helper_reach((graph_population.__name__, kernel.__name__))) - hashed - ) + reach, stops = _helper_walk((graph_population.__name__, kernel.__name__)) + missing = sorted(set(reach) - hashed) if missing: unhashed[kernel.ref] = missing + unlisted.update(item for item in stops if item[0] not in hashed) assert not unhashed, unhashed + assert unlisted == set(_DENSE_UNBOUND_NAMES), sorted( + unlisted ^ set(_DENSE_UNBOUND_NAMES) + ) + + +#: Adapters the spine build installs around a stage's own transform, each with +#: why it computes no value. +_SPINE_BUILD_ADAPTERS: dict[tuple[str, str], str] = { + ("microcosm.build.uk_runtime.spine_build", "_GraphSourceTransform"): ( + "builds the stage's own transform from the node's declared sources and calls it" + ), + ("microcosm.build.observation", "ObservedTransform"): ( + "times the wrapped transform's call and records its frames' sizes" + ), +} + + +def _spine_build_installs() -> list[tuple[str | None, tuple[str, str]]]: + """Each class or factory the production spine build installs as a + stage's transform, as ``(stage, (module, name))``, seen through the + adapters above and the local names it assigns; ``None`` for a wrapper it + installs around every stage.""" + + import ast + + from microcosm.build.uk_runtime import spine_build + + module = spine_build.__name__ + tree = ast.parse(Path(spine_build.__file__).read_text(encoding="utf-8")) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name == "prepare_uk_spine_execution" + ) + local: dict[str, list] = {} + for sub in ast.walk(function): + if ( + isinstance(sub, ast.Assign) + and len(sub.targets) == 1 + and isinstance(sub.targets[0], ast.Name) + ): + local.setdefault(sub.targets[0].id, []).append(sub.value) + installs: list[tuple[str | None, tuple[str, str]]] = [] + + def follow(expression, stage, seen=frozenset()) -> None: + if isinstance(expression, ast.Lambda): + follow(expression.body, stage, seen) + elif isinstance(expression, ast.Name) and expression.id not in seen: + for value in local.get(expression.id, ()): + follow(value, stage, seen | {expression.id}) + elif isinstance(expression, ast.Call) and isinstance(expression.func, ast.Name): + found = _resolve(module, expression.func.id, _HELPER_SCOPE) + if found is None: + return + installs.append((stage, found)) + if found in _SPINE_BUILD_ADAPTERS and stage and expression.args: + follow(expression.args[0], stage, seen) + + for sub in ast.walk(function): + if isinstance(sub, ast.Assign) and len(sub.targets) == 1: + target = sub.targets[0] + if isinstance(target, ast.Name) and target.id == "implementations": + if isinstance(sub.value, ast.Dict): + for key, value in zip( + sub.value.keys, sub.value.values, strict=True + ): + follow(value, key.value) + elif isinstance(sub.value, ast.DictComp): + follow(sub.value.value, None) + elif ( + isinstance(target, ast.Subscript) + and isinstance(target.value, ast.Name) + and target.value.id == "implementations" + and isinstance(target.slice, ast.Constant) + ): + follow(sub.value, target.slice.value) + return installs + + +def test_every_transform_the_spine_build_installs_is_hashed_with_its_stage() -> None: + """The production spine builds its stages' transforms in spine_build, not + in graph_kernels, so the rule above, which starts at graph_kernels' + transforms, does not see them. Each class or factory spine_build + installs for a stage is defined in the stage's module, declares the + modules it computes through (``graph_implementation_dependencies``, which + the stage's node hashes), or is an adapter listed with why it computes no + value (microcosm#931).""" + import importlib + + from microcosm.build.uk_runtime import graph_kernels + + installs = _spine_build_installs() + assert {stage for stage, _ in installs if stage} == set( + graph_kernels._STAGE_MODULES + ) + assert {found for stage, found in installs if stage is None} == { + ("microcosm.build.observation", "ObservedTransform") + } + unhashed = [] + for stage, (module, name) in installs: + if (module, name) in _SPINE_BUILD_ADAPTERS or ( + stage and module == graph_kernels._stage_module(stage).__name__ + ): + continue + declared = getattr( + getattr(importlib.import_module(module), name), + "graph_implementation_dependencies", + None, + ) + if not callable(declared): + unhashed.append((stage, f"{module}.{name}")) + assert not unhashed, unhashed + + +def test_the_sampled_spine_root_hashes_what_it_samples_through() -> None: + """The spine build wraps the root's transform in the rung sampler, which + samples the root below f100 and whose code lives in national_sampling and + the shared core; the root node then hashes every module the rule finds + the sampler computing through (microcosm#931).""" + from microcosm.build import frame_sampling + from microcosm.build.uk_runtime import graph_kernels, national_sampling + + sampler = national_sampling.UKSampledSpineRootTransform(None, fraction=0.1, seed=1) + plain = hashed_module_names(graph_kernels.UKCreateKernel(), graph_kernels) + hashed = hashed_module_names(graph_kernels.UKCreateKernel(sampler), graph_kernels) + assert hashed - plain == {national_sampling.__name__, frame_sampling.__name__} + reach = _helper_reach((national_sampling.__name__, type(sampler).__name__)) + assert {national_sampling.__name__, frame_sampling.__name__} <= set(reach) + assert set(reach) <= hashed, sorted(set(reach) - hashed) def test_uk_graph_json_round_trip_is_canonical() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 4227133ab3..fd959594ae 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1110,8 +1110,9 @@ def test_an_artifact_without_the_split_layer_is_refused(self) -> None: #: for the exact-total correction, and the zero-weight household is outside #: every pool. At 7 and 13 clones that household's clone 0 has no #: representable correction, so the shared fallback puts it on the last copy -#: of the 7,620 family's last clone, a few ulps off w / n and within the -#: 1e-12 tolerances; at 1, 3, 11 and 15 clones no correction is needed. +#: of the 7,620 family's last clone, off w / n by about an ulp of the frame's +#: total weight (a relative error below 1e-12, within the 1e-12 tolerances); +#: at 1, 3, 11 and 15 clones no correction is needed. _NESTING_SCENARIO = { "weights": [ 900.0, @@ -1175,6 +1176,10 @@ class TestSupportSplitRecomputeAcrossGeographicClones: same way. At one clone its receipt is the one the rule before ``K`` gave. At every ``K`` its count check holds the stored count to the family weights' relative tolerance of 1e-9, since the fold re-sums the copies. + Within that tolerance of a multiple of 60 it cannot tell ``n`` copies + from ``n + 1``: a stored count moved on its own is left to the flags and + the family weights, and a family split consistently into one copy more + goes unreported. """ def test_at_one_clone_the_receipt_is_the_rule_before_the_clone_count( @@ -1312,6 +1317,38 @@ def test_a_count_moved_on_an_exact_multiple_is_left_to_the_flags( } assert problems["support_split_family_weights"] == 15 + @pytest.mark.parametrize("clone_count", [1, 3]) + def test_a_family_split_into_one_more_copy_on_a_multiple_goes_unreported( + self, clone_count: int, monkeypatch + ) -> None: + # The blind spot the tolerance leaves (microcosm#931 review). A stage + # that counted floor(q) + 1 copies for q = K x w / 60 instead of + # ceil(q) differs only where q is an integer: here the four exact + # multiples (900, 1,800, 4,020 and 7,620), each split in full into one + # copy more, every member at w / (n + 1). No check reports it at any + # clone count; only the exact recount records the gap. + from microcosm.build.uk_runtime import cgt_support + + def floor_plus_one(weights, maximum_copy_weight, *, clone_count): + quotient = ( + clone_count + * np.asarray(weights, dtype=float) + / float(maximum_copy_weight) + ) + return np.maximum(np.floor(quotient).astype("int64") + 1, 1) + + with monkeypatch.context() as patched: + patched.setattr(cgt_support, "cgt_support_copy_counts", floor_plus_one) + split, frame = _clone_first_split_then_cloned( + clone_count, _NESTING_SCENARIO + ) + + receipt, problems = _exhausting_receipt(frame, _NESTING_SCENARIO) + + assert problems == {} + assert receipt["copies"] == split.copies_created == clone_count * (259 + 4) + assert receipt["copies_recomputed"] < receipt["copies"] + def test_at_one_clone_a_family_folded_just_above_a_multiple_is_green( self, ) -> None: @@ -1908,6 +1945,25 @@ def test_the_reversal_restores_what_stage_27_replaced(self) -> None: assert tool._pre_regional_home_values_frame(reversed_frame) is reversed_frame assert tool._pre_regional_home_values_frame(chain.split) is chain.split + @pytest.mark.parametrize("key", ["outputs", "rewrites"]) + def test_the_reversal_refuses_a_later_writer_of_the_values( + self, key: str, monkeypatch + ) -> None: + # The reversal assumes stage 27 writes the two columns last. A roster + # where a later stage declares one of them fails closed, instead of + # hiding that stage's rewrite from E5. + tool = _load_tool() + stages = tool._source_stage_roster() + names = [stage["stage"] for stage in stages] + later = names.index("regional_home_values") + 1 + edited = [dict(stage) for stage in stages] + edited[later][key] = [*edited[later].get(key, ()), "main_residence_value"] + valued = _home_values_chain(cgt=False).valued + assert tool._pre_regional_home_values_frame(valued) is not valued + monkeypatch.setattr(tool, "_source_stage_roster", lambda: edited) + with pytest.raises(ValueError, match=r"^1 stage\(s\) after regional_home"): + tool._pre_regional_home_values_frame(valued) + def test_dropping_the_copies_under_a_kept_cgt_layer_is_refused(self) -> None: tool = _load_tool() final = _home_values_chain(cgt=True).final @@ -1961,3 +2017,56 @@ def test_a_copy_off_its_lineage_fails_closed_by_count(self) -> None: tool._high_value_copy_lineage(tampered) assert str(caught.value) == message assert not any(str(int(value)) in str(caught.value) for value in copy_ids) + + +#: Row-creating stages whose declared flag the tool's stacking map has no +#: entry for, with why it needs none (microcosm#931 review). +_STACKING_MAP_EXCEPTIONS = { + "spi_income_band_donors": ( + "its donors are rewritten onto household_is_spi_synthetic too, so the " + "support channel's entry drops them" + ), + "cgt_incidence_anchor": ( + "declares only the incidence clone's flag, which it pairs on, and " + "creates no rows" + ), +} + + +def test_the_stacking_map_names_every_structural_stage_flag() -> None: + """The map's own comment says a new stacking stage must add its flag; this + holds it to the roster. Each structural stage's declared flag maps to + that stage, or the stage is an exception listed with its reason.""" + from microcosm.build.uk_runtime.graph import UK_SPINE_STRUCTURAL_STAGES + + tool = _load_tool() + stages = { + stage["stage"]: stage + for stage in tool._source_stage_roster() + if stage["stage"] in UK_SPINE_STRUCTURAL_STAGES + } + assert set(stages) == set(UK_SPINE_STRUCTURAL_STAGES) + declared = { + name: [ + operation["flag_column"] + for operation in stage.get("operations", ()) + if "flag_column" in operation + ] + for name, stage in stages.items() + } + mapped = {} + for name, flags in declared.items(): + if name in _STACKING_MAP_EXCEPTIONS: + continue + (flag,) = flags + mapped[flag] = name + assert mapped == tool._STACKING_STAGE_BY_FLAG + # The exceptions' reasons, as the roster declares them. + assert "household_is_spi_synthetic" in stages["spi_income_band_donors"].get( + "rewrites", () + ) + assert declared["cgt_incidence_anchor"] == [] + assert { + operation.get("clone_flag_column") + for operation in stages["cgt_incidence_anchor"]["operations"] + } - {None} == {"household_is_capital_gains_clone"} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py index b8eb24e138..80a5adc58e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_sampling.py @@ -283,3 +283,41 @@ def relabel_one_row_after_sampling(frame): assert str(caught.value) == ( "UK spine sample retained 1 incomplete source family(ies)." ) + + +def test_the_sample_node_calls_no_sampler_at_f100(monkeypatch) -> None: + """At f100 ``uk.full.sample`` keeps the spine whole and calls no sampler + (experiments/901-uk-main-rebase-receipts.md, row 4).""" + import json + from types import SimpleNamespace + + from microcosm.build import frame_sampling + from microcosm.build.uk_runtime import graph_population, national_sampling + + def no_sampler(*args, **kwargs): + raise AssertionError("uk.full.sample called a sampler at f100") + + monkeypatch.setattr(national_sampling, "sample_uk_spine_frame", no_sampler) + monkeypatch.setattr(national_sampling, "sample_frame_households", no_sampler) + monkeypatch.setattr(frame_sampling, "sample_frame_households", no_sampler) + frame = _spine_family_frame() + context = SimpleNamespace( + tables={entity: frame.table(entity) for entity in frame.entities}, + frame_column_order={}, + weights={"household": frame.weights_for("household")}, + strata=frame.strata, + params={"fraction": 1.0, "seed": 578, "time_period": "2023"}, + artifacts={}, + node=SimpleNamespace(artifact_inputs=()), + frame_mass_log=frame.mass_log, + frame_metadata=frame.metadata, + ) + + result = graph_population.UKFamilySampleKernel().run(context) + + payload = json.loads(result.artifacts["sampling"]) + assert payload["receipt"]["sampled"] is False + assert payload["receipt"]["rung_token"] == "f100" + assert bool(result.keep.all()) + assert payload["household_ids"] == frame.table("household")["household_id"].tolist() + assert payload["mass_log_append"] == [] diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 8158641b44..e5e7a84ace 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -853,11 +853,16 @@ def _e8_support_split( re-sums the ``n`` copies, which can land ulps from ``w``: at ``K = 1`` for a weight a few ulps above a multiple of the maximum, or where the stage's exact-total correction fell on a copy, and at ``K > 1`` by more - than the count's snap absorbs. So on an exact multiple the check accepts - ``n`` or ``n + 1``, and a count moved there is left to the flag and - family-weight checks. The receipt's ``copies_recomputed`` stays the exact - recount, so it can differ from ``copies`` by one for each family whose - re-summed weight lands across a multiple, while the count check passes. + than the count's snap absorbs. So for a family whose weight lies within + that tolerance of a multiple of the maximum, the check accepts ``n`` or + ``n + 1``. A stored count moved there on its own is still reported, by + the flag and family-weight checks, but a family the stage split + consistently into one copy more there, every member at ``w / (n + 1)`` + and indexed in full, is reported by no check at any ``K``. The receipt's + ``copies_recomputed`` stays the exact recount, so it can differ from + ``copies`` by one for each family whose re-summed weight lands across a + multiple, or that was split into one copy more, while the count check + passes. Every member of a family carries ``w / n`` at stage time (its clone pair sum, since the clone runs afterwards), the copies of each root are indexed ``1..n-1`` without gaps and carry the root's count, no count is @@ -1065,7 +1070,9 @@ def recompute(person_t, household_t, weights_t): # recount would flag a family the stage counted right, so at every K the # count is held to the family weights' tolerance instead: the stored n # must lie between the counts at the folded weight times 1 - 1e-9 and - # 1 + 1e-9 (microcosm#931 review). + # 1 + 1e-9 (microcosm#931 review). Within that tolerance of a multiple it + # cannot tell n from n + 1, so a family split consistently into one copy + # more there goes unreported. agreed_weights = pre_weights[agreed] lowest, highest = ( cgt_support_copy_counts( @@ -1791,7 +1798,9 @@ def plans_equal(left: pd.Series, right: pd.Series) -> bool: "the anchor each cost one float rounding generation); " "support-split mass record: new_total against old_total at rtol " "1e-9; support-split copy counts: between the counts at the " - "folded family weight times 1 - 1e-9 and 1 + 1e-9; support-split " + "folded family weight times 1 - 1e-9 and 1 + 1e-9, so within that " + "tolerance of a multiple of the maximum copy weight a family split " + "consistently into one copy more is not reported; support-split " "families, flags and student_loan_plan: exact equality; " "band-donor seats, bands and " "carriers: exact equality; band-donor weights within a band and " @@ -2088,7 +2097,8 @@ def main() -> int: #: is stored there — that value was drawn for the source id, not this one. #: A new stacking stage MUST add its flag here, or these receipts silently #: start comparing inherited values against fresh draws and report a spine -#: defect that is really an instrument defect. +#: defect that is really an instrument defect; a receipts test holds the map +#: to the structural stages' declared flags. _STACKING_STAGE_BY_FLAG = { # #717 SPI support channel; the #1006 income band donors carry the same flag. "household_is_spi_synthetic": "spi_support_channel", @@ -2864,6 +2874,42 @@ def _pre_salary_sacrifice_frame(frame): #: The stage that rewrites owners' home values in place (microcosm#1112). _REGIONAL_HOME_VALUES_STAGE = "regional_home_values" +#: The household columns that stage rewrites, which the reversal restores. +_REGIONAL_HOME_VALUE_COLUMNS = ("main_residence_value", "property_wealth") + + +def _source_stage_roster() -> list[dict[str, object]]: + """The committed source-stage roster's stages, in order.""" + + from importlib.resources import files as _files + + spec = json.loads( + _files("microcosm.build.uk") + .joinpath("source_stages.json") + .read_text(encoding="utf-8") + ) + return list(spec["stages"]) + + +def _require_regional_home_values_last_writer() -> None: + """Refuse when a stage after ``regional_home_values`` declares a column + the reversal restores: the reversal would then hide that stage's rewrite + from every receipt scoped before stage 27, instead of failing on it.""" + + stages = _source_stage_roster() + names = [str(stage["stage"]) for stage in stages] + later = [ + stage + for stage in stages[names.index(_REGIONAL_HOME_VALUES_STAGE) + 1 :] + if set(_REGIONAL_HOME_VALUE_COLUMNS) + & {*stage.get("outputs", ()), *stage.get("rewrites", ())} + ] + if later: + raise ValueError( + f"{len(later)} stage(s) after {_REGIONAL_HOME_VALUES_STAGE} declare " + "main_residence_value or property_wealth, so the reversal of its " + "rewrite no longer gives the values an earlier stage saw." + ) def _pre_regional_home_values_frame(frame): @@ -2882,7 +2928,9 @@ def _pre_regional_home_values_frame(frame): largest wealth the floor allows. The rank basis is dropped, since no earlier stage had it. An artifact without it (built before the stage, or a release, whose boundary drops it with ``property_wealth``) is returned - as it is. + as it is. The reversal holds only while stage 27 is the last stage to + write either column, so it refuses a roster where a later stage declares + one of them. """ from microcosm.build.uk_runtime.national_frame import ( @@ -2894,6 +2942,7 @@ def _pre_regional_home_values_frame(frame): household = frame.table("household") if RANK_BASIS_COLUMN not in household.columns: return frame + _require_regional_home_values_last_writer() household = household.copy() basis = pd.to_numeric(household[RANK_BASIS_COLUMN], errors="raise").to_numpy( dtype=float From c64c2b378c1d523f16ad3d1fc0a3eb774b33b1bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:07:39 +0000 Subject: [PATCH 026/108] Draw the SPI support channel as a keyed Bernoulli share (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 16 copied exactly round(10,000 x the sample fraction) households, picked by row position with rng.choice, so whether a household was copied depended on every other row. Once (b) clones households at the spine's start, the K=1 spine's copies would not be clone 0's copies at K=15. As María ruled on 7 Nov, the stage now copies a keyed Bernoulli share whose probability it computes itself. - Share: the transform passes expected = count x sample fraction, so the manifest's 10,000 is the share's numerator. create_uk_spi_support_tables takes expected_spi_households in place of spi_household_count (None still copies every household). It counts N, the distinct source_household_id values entering the stage, and copies a household when pi = min(1, expected / N) is one or the household's uniform falls below pi. The uniform is keyed on the household's own household_id, under the stage's seed (42) and the new salt spi_support_channel:inclusion. The decision is spi_share_inclusion, which takes pi explicitly; at pi = 1 it reads no uniform, since an integer over 2**64 can round to one. Copies keep frame order. The positional draw and the count-versus-rows refusal are gone. The tests-only replace path keeps its own draw. - Gate: e7_spi_synthetic_preclone is renamed e7_spi_synthetic_share. It declares maximum_zero_weight_rows_rule {rule: binomial_share_band, sigma: 4} in place of the integer. After the draw, the stage resolves the rule to min(n, ceil(n x (pi + band(pi, n)))) over the n household rows entering it, then runs the unchanged zero-weight gate. stage_health.latent_attribute_tolerance moved into stochastic_assignment as binomial_share_tolerance(*, target, rows, sigma). The stage_health name stays as an alias at four sigma, with unchanged values. _support_stage_parameters checks the draw, salt, gate name, selector, rule and reason, and returns the declaration with its rule. Its hmrc_source_contract caller needs no change. - Per-clone strata (ruling 2): where household_clone_index is present, _allocate_spi_prior_mass adds a derived clone stratum, which is never written to the frame. The manifest declares it as geographic_clone_stratum, and spi_spine checks it. The empty-stratum refusal now counts strata and names none. Without the column, as on every national spine today, the allocation is unchanged bit for bit. - Manifest and schema: the stage has a new source sentence, draw text, salt, gate reason and notes. In sources.schema.json, stack_zero_weight_donors requires salt, the allocation accepts geographic_clone_stratum, and a zero-weight declaration takes the integer maximum or the rule (oneOf). - Receipt: new keys expected_spi_households, inclusion_probability (receipt_counts.floored_share), source_households (rounded_count), and the gate's name, rule and sigma, not its resolved maximum. spi_households stays exact, because the stage gate reads it. - Bindings: the receipt's counting sits in spi_spine. That adds one counting stage (frs_hmrc_spine_leaves), against three for spi_support (cgt_support_split, cgt_incidence_clone, cgt_residential_split), which ten stages bind. Node re-keys are the same either way: the root already binds receipt_counts. high_value_home_support and cgt_residential_split now bind stochastic_assignment, as they bind spi_support. What moves: at K=1 the SPI copy set is redrawn. Its count is Binomial(N, pi) around 10,000 x f instead of exact: about 1,000 at f010, by design arithmetic. Per-stratum SPI weights move with it, and so do every SPI-channel value from stage 18 on, the high-value and CGT selections that rank SPI rows, and calibration. The claim that stage 17's seating does not move holds on synthetic frames. On 8 frames run through the parent's stage 16 and this one, stage 17 seated the same households, carriers and bands, with equal seating scales, mass scales and funding factors. Stage 16's base weights were bitwise equal except for one row in one frame (relative 7e-14, the exact-total correction). The H2 fixture's 135 households at f = 135/10,000 give pi = 1 (10,000 x 0.0135 is 135.0 in float), so its copies are unchanged. Every node re-keys, since graph_kernels, stochastic_assignment and spi_support changed and the root binds stochastic_assignment; so does every kernel keyed on the spec or the coverage manifest. Regenerated: source_stages.json, the coverage manifest's 25 source digests, the SPI declarations in the H2 fixture (fixture.json), and the stage's contract hash (uk_spine.json). Tests: - With pi held fixed, permuting rows or appending other households leaves every household's draw unchanged. A row permutation leaves the stage's selection unchanged. - Over 200 seeds the count stays inside the 4-sigma band. - pi = 1 copies every household, even with uniforms patched to 1.0. - The resolved gate passes at its maximum and refuses one more row. The stage gate refuses an injected copy above the bound, and an unmatched zero-weight base row still fails. - Malformed rules, and drift in the draw, salt, rule, sigma, name or clone stratum, are refused. - On clone-first frames at K=3 and 7, N is the same as at K=1, and clone 0's copies are exactly K=1's. The other clones draw their own, and K x clone-0 weights equal K=1's to rtol 1e-12. - Each clone keeps its own region masses. An empty clone stratum is refused by count, and so are a non-integer, missing or negative clone index. A one-clone index allocates bit for bit as no index. - The band-donor fixture moves to expected_spi_households=10, and a comment records that seed 42 keeps copies in both regions at 60 and 200 households. Its weight test now takes the median over the FRS base the seats are drawn from: over every row, the redrawn copies moved the median and the statistic fell just under its 0.53 bar, while the seating itself is unchanged. - Ten in-memory mutations each fail a new test: keying on the source id, a row-count denominator, no pi = 1 short-circuit, no clone stratum, the positional draw, a gate without its band, a three-sigma alias, an exact share or source count in the receipt, and another salt. The touched engine-free files pass, with ruff. The coverage manifest is current. The H2 fixture regenerates and passes its check, and H2 parity passes in the engine slot. Co-Authored-By: Claude Opus 5.5 --- .../931-spi-support-keyed-share.changed.md | 7 + .../spec_engine/schema/sources.schema.json | 37 +- .../microcosm/build/stochastic_assignment.py | 20 + .../uk/release_input_coverage_manifest.json | 50 +- .../src/microcosm/build/uk/source_stages.json | 21 +- .../src/microcosm/build/uk/spec/sources.yaml | 16 +- .../build/uk_runtime/graph_kernels.py | 14 +- .../microcosm/build/uk_runtime/spi_spine.py | 145 +++-- .../microcosm/build/uk_runtime/spi_support.py | 324 ++++++++-- .../build/uk_runtime/stage_health.py | 19 +- .../shared/test_stochastic_assignment.py | 43 ++ .../engine_free/uk/test_uk_copy_id_table.py | 2 +- .../tests/engine_free/uk/test_uk_frs_spine.py | 12 +- .../uk/test_uk_hmrc_income_source_manifest.py | 36 ++ .../engine_free/uk/test_uk_source_stages.py | 6 + .../engine_free/uk/test_uk_spi_band_donors.py | 15 +- .../tests/engine_free/uk/test_uk_spi_spine.py | 241 ++++++- .../engine_free/uk/test_uk_spi_support.py | 598 +++++++++++++++++- .../parity/uk_spine/sources/fixture.json | 17 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- 20 files changed, 1426 insertions(+), 199 deletions(-) create mode 100644 changelog.d/931-spi-support-keyed-share.changed.md diff --git a/changelog.d/931-spi-support-keyed-share.changed.md b/changelog.d/931-spi-support-keyed-share.changed.md new file mode 100644 index 0000000000..6a9f297bef --- /dev/null +++ b/changelog.d/931-spi-support-keyed-share.changed.md @@ -0,0 +1,7 @@ +The SPI support channel (`spi_support_channel`) copies its zero-weight SPI households as a keyed Bernoulli share instead of a fixed count drawn by row position (#931). A household is copied when its uniform, keyed on its own `household_id` under the stage's seed (42) and the new salt `spi_support_channel:inclusion`, falls below π = min(1, 10,000 × the sample fraction / the distinct `source_household_id` values entering the stage); at π = 1 every household is copied without reading a uniform. The manifest's 10,000 is now the share's numerator, so a one-clone spine's copy count is binomial, with mean min(10,000 × the sample fraction, the source households), instead of exact: about 1,000 at f010, by design arithmetic (10,000 × 0.1). Each household's draw reads only its own id, the seed, the salt and π, so row order does not matter. π counts true sources, so it is the same at every geographic clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, draws exactly the one-clone copies, while the other clones draw on their own ids (María's rulings of 7 Nov). + +- **Gate.** `e7_spi_synthetic_preclone` is renamed `e7_spi_synthetic_share`. Its declaration carries `maximum_zero_weight_rows_rule: {rule: binomial_share_band, sigma: 4}` instead of the integer 10,000. After the draw the stage resolves it to min(n, ceil(n × (π + band(π, n)))) over the n household rows entering the stage, and runs the unchanged zero-weight gate. The band is the repo's binomial band, moved unchanged from `stage_health.latent_attribute_tolerance` into `stochastic_assignment.binomial_share_tolerance(*, target, rows, sigma)`; `latent_attribute_tolerance` stays, as the same band at four sigma. In `sources.schema.json`, `stack_zero_weight_donors` requires `salt`, a zero-weight declaration takes either the integer maximum or the rule, and `allocate_zero_weight_prior_mass` accepts `geographic_clone_stratum`. The stage's source sentence, draw text and notes describe the share. +- **Per-clone strata.** Where the household table carries `household_clone_index`, the prior allocation's strata also split by geographic clone, so each clone keeps its own mass and its SPI weights depend on its own households and copies only. The manifest declares this as `geographic_clone_stratum: household_clone_index`. A positive-mass stratum without an SPI copy is refused with a count of strata, not their names. Without the column, as on every national spine today, the allocation is unchanged bit for bit. The small-clone-stratum fallback comes later in #931. +- **Receipt.** The stage evidence gains `expected_spi_households` (10,000 × the sample fraction), `inclusion_probability` floored to two decimals, `source_households` rounded by the receipts' rule, and the gate's name, rule and sigma, not the maximum it resolved to. `spi_households` stays exact, because the stage gate reads it. +- **Bindings.** spi_spine, the module of stages 14, 16 and 18, rounds the receipt, so `frs_hmrc_spine_leaves` joins the counting stages. spi_support now draws through the keyed-draw helpers, so `high_value_home_support` and `cgt_residential_split`, which bind it, bind them too. +- **What moves.** At K = 1 the SPI copy set is redrawn, with a binomial count around 10,000 × the sample fraction where that is below the source households. Per-stratum SPI weights move with it, and so does every SPI-channel value from `hmrc_spi_income_spine` on, the high-value and CGT selections that rank SPI rows, and calibration. Base FRS rows keep their stage-16 weights, which are their incoming weights times stratum shares that do not depend on the draw, except the row the exact-total correction lands on. On synthetic frames, `spi_income_band_donors` seats the same households, carriers and bands under the old and new draws. The H2 fixture's 135 households at a sample fraction of 135 / 10,000 give π = 1 (10,000 × 0.0135 is 135.0 in float), so its copies are unchanged; its declarations and the stage's contract hash are regenerated, with the coverage manifest's source digests and `source_stages.json`. Every node re-keys. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 91fc03a3ed..4b5e283890 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -8272,6 +8272,7 @@ "draw", "flag_column", "id_remapping", + "salt", "seed" ], "properties": { @@ -8290,6 +8291,9 @@ "flag_column": { "type": "string" }, + "salt": { + "type": "string" + }, "seed": { "type": "integer" }, @@ -8335,9 +8339,20 @@ "required": [ "name", "selector", - "maximum_zero_weight_rows", "reason" ], + "oneOf": [ + { + "required": [ + "maximum_zero_weight_rows" + ] + }, + { + "required": [ + "maximum_zero_weight_rows_rule" + ] + } + ], "properties": { "name": { "type": "string" @@ -8348,6 +8363,23 @@ "maximum_zero_weight_rows": { "type": "integer" }, + "maximum_zero_weight_rows_rule": { + "type": "object", + "additionalProperties": false, + "required": [ + "rule", + "sigma" + ], + "properties": { + "rule": { + "const": "binomial_share_band" + }, + "sigma": { + "type": "number", + "exclusiveMinimum": 0 + } + } + }, "selector": { "type": "object", "minProperties": 1, @@ -8381,6 +8413,9 @@ "conservation": { "type": "string" }, + "geographic_clone_stratum": { + "type": "string" + }, "pension_age_share": { "type": "number" }, diff --git a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py index 2d73825ed7..dee464c78d 100644 --- a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py +++ b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import math from collections.abc import Mapping, Sequence from typing import Any @@ -12,6 +13,7 @@ __all__ = [ "assign_binary_from_rate", "assign_binary_with_anchored_residual", + "binomial_share_tolerance", "clipped_normal_from_uniforms", "integer_record_keys", "sample_categorical_from_counts", @@ -125,6 +127,24 @@ def integer_record_keys( return keys +def binomial_share_tolerance(*, target: float, rows: int, sigma: float) -> float: + """Binomial band for a share realized over ``rows`` identity-keyed draws. + + ``sigma`` standard errors of a share whose rows are each drawn at + ``target``, ``sigma x sqrt(target x (1 - target) / rows)``, floored at one + row's worth and capped at one; one when there are no rows. Moved here + from the UK stage-health gates, which keep their four-sigma band under + their own name (microcosm#931). + """ + + if rows <= 0: + return 1.0 + return min( + 1.0, + max(1.0 / rows, sigma * math.sqrt(target * (1.0 - target) / rows)), + ) + + def assign_binary_from_rate( draws: Sequence[float] | np.ndarray, rate: float, diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 3c84db5f8c..13f6baeda3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "51521b3193527bbcd4ae2ecdee90e2f0ce9c6d1379446977f7950e14576ccaae", + "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 2b9b8798ef..fbd58cc484 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -1671,7 +1671,7 @@ { "stage": "spi_support_channel", "survey": "Family Resources Survey 2024-25", - "source": "Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning.", + "source": "Synthetic SPI support channel drawn from the raw FRS spine as a keyed Bernoulli share, each household on its own identity-keyed uniform.", "grain": "household", "artifacts": [], "operations": [ @@ -1684,20 +1684,24 @@ "base": "frs", "synthetic": "spi" }, - "draw": "uniform_without_replacement", - "id_remapping": "copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11)" + "draw": "keyed_bernoulli_share: stable_identity_uniforms(household_id, seed, salt) < min(1, count x sample fraction / distinct source households)", + "id_remapping": "copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11)", + "salt": "spi_support_channel:inclusion" }, { "kind": "gate_zero_weight_strata", "gate": "uk_zero_weight_strata_gate", "declarations": [ { - "name": "e7_spi_synthetic_preclone", + "name": "e7_spi_synthetic_share", "selector": { "household_is_spi_synthetic": true }, - "maximum_zero_weight_rows": 10000, - "reason": "E7 stacks exactly 10,000 zero-weight SPI-synthetic households on the raw pre-clone UK FRS spine." + "reason": "E7 stacks a keyed Bernoulli share of the raw UK FRS spine's households as zero-weight SPI-synthetic households, so their count is binomial; it may exceed the share of the household rows entering the stage by at most the repo's four-sigma binomial band, floored at one row.", + "maximum_zero_weight_rows_rule": { + "rule": "binomial_share_band", + "sigma": 4 + } } ] }, @@ -1710,7 +1714,8 @@ "pension_age_share": 0.2, "pension_age_stratum": "any_member_at_or_over_state_pension_age", "weight_kind_out": "importance", - "conservation": "exact_total" + "conservation": "exact_total", + "geographic_clone_stratum": "household_clone_index" } ], "outputs": [ @@ -1738,7 +1743,7 @@ "source_household_id", "source_year" ], - "notes": "Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates prior mass with exact national conservation within each region, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." + "notes": "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's draw depends on its own id alone. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." }, { "stage": "spi_income_band_donors", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 88fe7c56ec..391c83c596 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -1511,31 +1511,35 @@ stages: notes: 'Describes each FRS adult''s 12 months before the interview beside the current income columns, which it never changes, so the HMRC income-band measures can read annual income while poverty, benefits and the published engine tax keep current income. HMRC counts income received over a tax year: a person who started, left or changed a job, or had a gap, sits in another HMRC band than current pay puts them in, or in none. Outputs: the share of the year paid at today''s employee pay and annual pay from other spells (chiefly a recent leaver''s last job), the same pair for self-employment, and the shares of the year a private pension started on retiring and contributory JSA, ESA and Incapacity Benefit started on leaving work were in payment; an annual amount is the current amount times its share plus the other spells. The State Pension needs none: policyengine-uk''s months_since_state_pension_age gives its share. Months come from CURRJOBM and JOB.WRKPREV for people in work (a person in work is never a leaver), the SAMESIT/SDEMP calendar only where it records a change, and LSTYR, LSTWRK1/2 or RETIRE1 for people out of work who worked in the last 12 months (NUMJOB2 and their last-job JOB rows); a leaving date 12 months or more before the interview gives none. A leaver''s pay is the last job''s usual gross pay (JOB.UGROSS, asked of people out of work who last worked in the previous 12 months, in respect of that job), scaled to the DV concept by the weighted job-level UGRSPAY/UGROSS ratio on current employees who report both and uprated by the pinned OBR average earnings index for the months since leaving; otherwise a donor''s weekly pay, drawn by GROSS4 weight from current employees in their first 24 months in the job within age band x sex x full- or part-time cells (sex dropped below 30 donors), on an identity-keyed uniform (seed 0, salt employment_income_other_spells). Self-employment keeps the whole year where SEJBLONG is one or more, (SEJBMTHS + 1)/12 under a year and at most the calendar''s self-employed months where self-employment is the activity today (the calendar records the main activity only, so a side business beside a job keeps its whole year); a ceased self-employed last job pays SEINCAMT (else PROFIT1, a loss as zero) for its SELWKS weeks; SEINCAM2 is already uprated to the interview by the FRS, so it is not uprated again. The receipt gives each source''s support as a band of distinct households, ratios only where three or more households carry them, and the exact zeros its stage gate reads.' - stage: spi_support_channel survey: Family Resources Survey 2024-25 - source: Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning. + source: Synthetic SPI support channel drawn from the raw FRS spine as a keyed Bernoulli share, each household on its own identity-keyed uniform. grain: household artifacts: [] operations: - kind: stack_zero_weight_donors count: 10000 seed: 42 + salt: spi_support_channel:inclusion flag_column: household_is_spi_synthetic channels: base: frs synthetic: spi - draw: uniform_without_replacement + draw: 'keyed_bernoulli_share: stable_identity_uniforms(household_id, seed, salt) < min(1, count x sample fraction / distinct source households)' id_remapping: copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11) - kind: gate_zero_weight_strata gate: uk_zero_weight_strata_gate declarations: - - name: e7_spi_synthetic_preclone + - name: e7_spi_synthetic_share selector: household_is_spi_synthetic: true - maximum_zero_weight_rows: 10000 - reason: E7 stacks exactly 10,000 zero-weight SPI-synthetic households on the raw pre-clone UK FRS spine. + maximum_zero_weight_rows_rule: + rule: binomial_share_band + sigma: 4 + reason: E7 stacks a keyed Bernoulli share of the raw UK FRS spine's households as zero-weight SPI-synthetic households, so their count is binomial; it may exceed the share of the household rows entering the stage by at most the repo's four-sigma binomial band, floored at one row. - kind: allocate_zero_weight_prior_mass share: 0.5 strata: - region + geographic_clone_stratum: household_clone_index pension_age_share: 0.2 pension_age_stratum: any_member_at_or_over_state_pension_age weight_kind_out: importance @@ -1563,7 +1567,7 @@ stages: - household_source_id - source_household_id - source_year - notes: "Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates prior mass with exact national conservation within each region, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." + notes: "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's draw depends on its own id alone. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." - stage: spi_income_band_donors survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5 source: Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane). diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 937702bd38..7707ce8e9f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -350,8 +350,9 @@ trace, stochastic_assignment, ), - # The support channel reads the take-up population's ages and gates its - # zero-weight strata (microcosm#931). + # The support channel draws its keyed share and bounds its zero-weight + # gate through the keyed-draw helpers, reads the take-up population's ages + # and gates its zero-weight strata (microcosm#931). "spi_support_channel": ( spi_support, copy_id_table, @@ -372,13 +373,16 @@ ), # The exact-total correction the CGT split hands the high-value split is # spi_support's, and the support stratum is property_localisation's, whose - # owners are the tenure map's (microcosm#931). + # owners are the tenure map's (microcosm#931). spi_support draws the SPI + # share through the keyed-draw helpers, so the stages that bind it bind + # them too. "high_value_home_support": ( cgt_support, copy_id_table, property_localisation, spi_support, tenure_constants, + stochastic_assignment, ), # The CGT stages read HMRC's CGT tables as vendored facts and the # Advani-Summers tail from advani_summers, take the CGT policy parameters @@ -422,6 +426,7 @@ cgt_imputation, hmrc_capital_gains, spi_support, + stochastic_assignment, ), "hmrc_cgt_gains_spine": ( *_FEED_FACT_MODULES, @@ -772,6 +777,9 @@ "frs_take_up", "frs_person_draws", "frs_household_draws", + # spi_spine, stage 14's module, rounds the support channel's receipt + # (microcosm#931). + "frs_hmrc_spine_leaves", "frs_annual_income_spells", "hmrc_spi_income_spine", "spi_support_channel", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index ebd729a0bb..e33b882a6f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -55,6 +55,7 @@ declared_gate_calibration, declared_gate_level_columns, ) +from microcosm.build.uk_runtime.receipt_counts import floored_share, rounded_count from microcosm.build.uk_runtime.release_input_coverage import ( DEFAULT_MINIMUM_NONDEFAULT_MASS_SHARE, ) @@ -108,6 +109,7 @@ DEFAULT_SPI_SUPPORT_HOUSEHOLDS, FRS_ONLY_SPI_FILL_INCOME_PREDICTOR_COLUMNS, HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, + SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN, SPI_HMRC_DERIVED_AUXILIARY_COLUMNS, SPI_HMRC_EMPLOYMENT_BENEFITS_COLUMN, SPI_HMRC_EMPLOYMENT_EXPENSES_COLUMN, @@ -119,7 +121,9 @@ SPI_HMRC_TOTAL_EARNED_INCOME_COLUMN, SPI_HMRC_TOTAL_INVESTMENT_INCOME_COLUMN, SPI_INCOME_QRF_OUTPUT_COLUMNS, + SPI_SHARE_GATE_RULE, SPI_SUPPORT_ID_REMAPPING, + SPI_SUPPORT_INCLUSION_SALT, SPI_SYNTHETIC_SUPPORT_CHANNEL, UKSPISupportResult, build_uk_spi_support_channel, @@ -130,9 +134,6 @@ from microcosm.build.uk_runtime.spi_support import ( _spi_prior_mass_change_reason as _support_mass_change_reason, ) -from microcosm.build.uk_runtime.terminal_gates import ( - UKZeroWeightStratumDeclaration, -) from microcosm.frame import Frame, WeightKind, engine_tables UK_FRS_HMRC_SPINE_LEAVES_STAGE_NAME = "frs_hmrc_spine_leaves" @@ -397,7 +398,16 @@ "charitable_investment_gifts": 0.0, } SPI_SPINE_SUPPORT_CHANNELS = {"base": "frs", "synthetic": SPI_SYNTHETIC_SUPPORT_CHANNEL} -SPI_SPINE_PRECLONE_GATE_NAME = "e7_spi_synthetic_preclone" +#: The support channel's draw, as the manifest restates it: a keyed Bernoulli +#: share whose numerator is the declared count (microcosm#931). +SPI_SPINE_SUPPORT_DRAW = ( + "keyed_bernoulli_share: stable_identity_uniforms(household_id, seed, salt) " + "< min(1, count x sample fraction / distinct source households)" +) +SPI_SPINE_SHARE_GATE_NAME = "e7_spi_synthetic_share" +#: The share gate's bound: the repo's four-sigma binomial band above the +#: share (the latent-attribute receipts' sigma; ruled 2026-11-07). +SPI_SPINE_SHARE_GATE_RULE = {"rule": SPI_SHARE_GATE_RULE, "sigma": 4.0} #: The reviewed SPI-channel prior share of households with a member at or over #: State Pension age, and the stratum rule the manifest declares beside it #: (microcosm#1069 c6). The channel's households are drawn unweighted with one @@ -647,9 +657,9 @@ def checkpoint_metadata(self) -> dict[str, object]: class UKSPISupportChannelStageTransform: stage: SourceStageSpec seed: int = 42 - # A #627 scale-ladder receipt build: scale the declared synthetic stack - # by the survey-side sample fraction so the prior-mass pairing holds at - # every rung (f100 keeps the declared count exactly). + # A #627 scale-ladder receipt build: the declared count times the + # survey-side sample fraction is the share's expected copy count, so the + # prior-mass pairing holds at every rung in expectation (microcosm#931). sample_fraction: float = 1.0 # The engine's State Pension age at the build instant; None reads it from # policyengine-uk (tests without the engine pass it). @@ -663,8 +673,10 @@ def __call__(self, frame: Frame) -> Frame: count, share, strata, declarations, pension_age_share = ( _support_stage_parameters(self.stage, seed=self.seed) ) - if self.sample_fraction != 1.0 and count is not None: - count = max(1, int(round(count * self.sample_fraction))) + # The declared count is the share's numerator: each household is + # copied with probability min(1, count x sample fraction / distinct + # source households), computed by the builder (microcosm#931). + expected = float(count) * float(self.sample_fraction) state_pension_age = None if pension_age_share is not None: state_pension_age = ( @@ -678,7 +690,7 @@ def __call__(self, frame: Frame) -> Frame: tables["person"], tables["benunit"], tables["household"], - spi_household_count=count, + expected_spi_households=expected, seed=self.seed, source_year=int(uk_time_period(frame)), spi_prior_mass_share=share, @@ -721,23 +733,46 @@ def output_columns() -> tuple[str, ...]: def checkpoint_metadata(self) -> dict[str, object]: if self.last_result is None: raise RuntimeError("checkpoint metadata requires a completed stage run.") + result = self.last_result + if ( + result.spi_inclusion_probability is None + or result.spi_source_households is None + ): + raise RuntimeError("The SPI support stage records a drawn share.") return { "evidence": { "stage": "spi_support_channel", - "spi_households": self.last_result.n_spi_households, - "spi_prior_mass_share": self.last_result.spi_prior_mass_share, + # The copy count is exact: the stage gate reads it, and it + # counts a draw over every household, not an attribute cell. + "spi_households": result.n_spi_households, + # The share (microcosm#931): its expected count is the declared + # count times the sample fraction; the probability and the + # source households it divides by follow the receipts' + # disclosure rules, since an exact probability gives the + # source count back; and the gate's bound is recorded as its + # rule, not the maximum it resolved to. + "expected_spi_households": result.expected_spi_households, + "inclusion_probability": floored_share( + result.spi_inclusion_probability + ), + "source_households": rounded_count(result.spi_source_households), + "zero_weight_gate": { + "name": SPI_SPINE_SHARE_GATE_NAME, + **SPI_SPINE_SHARE_GATE_RULE, + }, + "spi_prior_mass_share": result.spi_prior_mass_share, "pension_age_spi_prior_mass_share": ( - self.last_result.pension_age_spi_prior_mass_share + result.pension_age_spi_prior_mass_share ), - "state_pension_age": self.last_result.state_pension_age, + "state_pension_age": result.state_pension_age, "pension_age_allocation": ( None - if self.last_result.pension_age_allocation is None - else dict(self.last_result.pension_age_allocation) + if result.pension_age_allocation is None + else dict(result.pension_age_allocation) ), "household_weight_kind": ( - self.last_result.household_weight_kind.value - if self.last_result.household_weight_kind is not None + result.household_weight_kind.value + if result.household_weight_kind is not None else None ), } @@ -1179,9 +1214,17 @@ def _support_stage_parameters( int, float, tuple[str, ...], - tuple[UKZeroWeightStratumDeclaration, ...], + tuple[Mapping[str, object], ...], float | None, ]: + """The support stage's reviewed parameters, checked against the code. + + Returns the declared count (the share's numerator), the prior-mass share, + the allocation strata, the zero-weight gate's one declaration with its + maximum rule (resolved after the draw, microcosm#931) and the pension-age + share. + """ + stack = _operation(stage, "stack_zero_weight_donors") gate = _operation(stage, "gate_zero_weight_strata") allocation = _operation(stage, "allocate_zero_weight_prior_mass") @@ -1189,8 +1232,12 @@ def _support_stage_parameters( raise ValueError("SPI support manifest count drifted from the reviewed value.") if stack.parameters.get("seed") != seed: raise ValueError("SPI support manifest seed drifted from the reviewed value.") - if stack.parameters.get("draw") != "uniform_without_replacement": - raise ValueError("SPI support manifest must declare a uniform donor draw.") + if stack.parameters.get("draw") != SPI_SPINE_SUPPORT_DRAW: + raise ValueError( + "SPI support manifest must declare the keyed Bernoulli share draw." + ) + if stack.parameters.get("salt") != SPI_SUPPORT_INCLUSION_SALT: + raise ValueError("SPI support manifest salt drifted from the reviewed value.") if stack.parameters.get("id_remapping") != SPI_SUPPORT_ID_REMAPPING: raise ValueError( "SPI support manifest id_remapping drifted from the copy-id table's " @@ -1211,6 +1258,14 @@ def _support_stage_parameters( strata = tuple(allocation.parameters.get("strata", ())) if strata != ("region",): raise ValueError("SPI support spine allocation must use strata ['region'].") + if ( + allocation.parameters.get("geographic_clone_stratum") + != SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN + ): + raise ValueError( + "SPI support allocation must split its strata by geographic clone " + f"on {SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN!r}." + ) pension_age_share = allocation.parameters.get("pension_age_share") if pension_age_share is not None: if pension_age_share != SPI_SPINE_PENSION_AGE_PRIOR_MASS_SHARE: @@ -1227,24 +1282,39 @@ def _support_stage_parameters( ) elif "pension_age_stratum" in allocation.parameters: raise ValueError("SPI support pension-age stratum declared without a share.") - declarations = _coerce_declarations(gate.parameters.get("declarations", ())) - if len(declarations) != 1: - raise ValueError("SPI support gate must declare exactly one pre-clone stratum.") - declaration = declarations[0] + declarations = gate.parameters.get("declarations", ()) if ( - declaration.name != SPI_SPINE_PRECLONE_GATE_NAME - or dict(declaration.selector) != {HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True} - or declaration.maximum_zero_weight_rows != DEFAULT_SPI_SUPPORT_HOUSEHOLDS + not isinstance(declarations, Sequence) + or isinstance(declarations, str | bytes) + or len(declarations) != 1 + or not isinstance(declarations[0], Mapping) + ): + raise ValueError("SPI support gate must declare exactly one share stratum.") + declaration = dict(declarations[0]) + rule = declaration.get("maximum_zero_weight_rows_rule") + reason = declaration.get("reason") + if ( + set(declaration) + != {"name", "selector", "maximum_zero_weight_rows_rule", "reason"} + or declaration["name"] != SPI_SPINE_SHARE_GATE_NAME + or not isinstance(declaration["selector"], Mapping) + or dict(declaration["selector"]) != {HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True} + or not isinstance(rule, Mapping) + or dict(rule) != SPI_SPINE_SHARE_GATE_RULE + or not isinstance(reason, str) + or not reason.strip() ): raise ValueError( - "SPI support pre-clone gate declaration drifted from the reviewed " - "name/selector/maximum." + "SPI support share gate declaration drifted from the reviewed " + "name/selector/maximum rule." ) + declaration["selector"] = dict(declaration["selector"]) + declaration["maximum_zero_weight_rows_rule"] = dict(rule) return ( int(stack.parameters["count"]), float(allocation.parameters["share"]), strata, - declarations, + (declaration,), None if pension_age_share is None else float(pension_age_share), ) @@ -1545,19 +1615,6 @@ def _assert_identity_matches_artifact( ) -def _coerce_declarations( - values: object, -) -> tuple[UKZeroWeightStratumDeclaration, ...]: - if not isinstance(values, Sequence) or isinstance(values, str | bytes): - raise ValueError("zero-weight declarations must be a sequence.") - declarations = [] - for value in values: - if not isinstance(value, Mapping): - raise ValueError("zero-weight declarations must be mappings.") - declarations.append(UKZeroWeightStratumDeclaration(**dict(value))) - return tuple(declarations) - - __all__ = [ "EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN", "UK_FRS_HMRC_SPINE_LEAF_OUTPUT_COLUMNS", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py index 10d01a8ee4..30f2455d01 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py @@ -9,6 +9,7 @@ from __future__ import annotations +import math from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace from typing import Any @@ -16,6 +17,11 @@ import numpy as np import pandas as pd +from microcosm.build.stochastic_assignment import ( + binomial_share_tolerance, + integer_record_keys, + stable_identity_uniforms, +) from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.hmrc_income import ( HMRC_SPI_ASSESSABLE_INCOME_COLUMN, @@ -41,7 +47,28 @@ "copy-id table field spi_support_channel; the SPI copy takes every entity " "id + 1 x the field's radix (10**11)" ) +#: The SPI share's numerator: the channel copies each household with +#: probability min(1, this x the rung's sample fraction / the distinct source +#: households entering the stage), so it holds about this many copies at full +#: scale (microcosm#931). DEFAULT_SPI_SUPPORT_HOUSEHOLDS = 10_000 +#: The salt the share's inclusion uniforms are keyed under, with the stage's +#: declared seed, on each household's own id (microcosm#931). +SPI_SUPPORT_INCLUSION_SALT = "spi_support_channel:inclusion" +#: The rule the share's zero-weight gate is bounded by: the share plus the +#: repo's binomial band at the declared sigma +#: (:func:`~microcosm.build.stochastic_assignment.binomial_share_tolerance`). +SPI_SHARE_GATE_RULE = "binomial_share_band" +#: The geographic clone index the clone-first spine gives every household +#: (``rowwise_dataset.ladder_clone_index_column("household")``, the +#: ``geographic_support`` branch of the UK copy lineage; microcosm#931). Where +#: the frame carries it, the prior allocation's strata split by it, so each +#: clone keeps its own mass. It is spelled here, not imported, so a change to +#: it re-keys the stage; a test holds the spellings equal. +SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN = "household_clone_index" +#: The allocation stratum derived from it, for the allocation only and never +#: written to the frame. +SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN = "household_geographic_clone_stratum" DEFAULT_SPI_PRIOR_MASS_SHARE = 0.5 HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN = "household_is_spi_synthetic" SPI_REPLACEMENT_STRATA_COLUMNS = ( @@ -251,6 +278,13 @@ class UKSPISupportResult: pension_age_spi_prior_mass_share: float | None = None state_pension_age: int | None = None pension_age_allocation: Mapping[str, float] | None = None + #: The share's draw (microcosm#931): the expected copy count it was given, + #: the distinct source households it counted and the probability it drew + #: at. Copying every household draws at one and counts nothing; named + #: copies (the tests-only replace path) record none of the three. + expected_spi_households: float | None = None + spi_source_households: int | None = None + spi_inclusion_probability: float | None = None @property def n_spi_households(self) -> int: @@ -262,7 +296,7 @@ def create_uk_spi_support_tables( person: pd.DataFrame, benunit: pd.DataFrame, household: pd.DataFrame, - spi_household_count: int | None = DEFAULT_SPI_SUPPORT_HOUSEHOLDS, + expected_spi_households: float | None = DEFAULT_SPI_SUPPORT_HOUSEHOLDS, seed: int = 42, source_year: int | None = None, selected_household_ids: Sequence[Any] | None = None, @@ -276,6 +310,17 @@ def create_uk_spi_support_tables( that lineage stable prevents long local-geography support summaries from counting the SPI copy as additional independent FRS support. + The copies are a keyed Bernoulli share (microcosm#931): with ``pi = min(1, + expected_spi_households / N)``, where ``N`` counts the distinct + ``source_household_id`` values of the incoming households (their true + sources, so geographic clones count once), each household is copied when + ``pi`` is one or its uniform, keyed on its own ``household_id`` under + ``seed`` and :data:`SPI_SUPPORT_INCLUSION_SALT`, falls below ``pi`` + (:func:`spi_share_inclusion`). The copy count is binomial around + ``expected_spi_households`` per clone, and the copies keep frame order. + ``None`` copies every household; ``selected_household_ids`` names them + instead (the tests-only replace path). + Each copy's five id columns take ``id + radix``, the declared radix of the copy-id table's ``spi_support_channel`` field (:data:`SPI_SUPPORT_ID_REMAPPING`), so every incoming id must sit below it (microcosm#931). @@ -301,16 +346,23 @@ def create_uk_spi_support_tables( values, SPI_SUPPORT_COPY_ID_FIELD, what=f"incoming {column} values" ) - if selected_household_ids is None: - selected_household_ids = _sample_spi_household_ids( + expected = source_households = inclusion_probability = None + if selected_household_ids is not None: + selected_household_ids = _validate_selected_household_ids( household_frame, - spi_household_count=spi_household_count, - seed=seed, + selected_household_ids, ) + elif expected_spi_households is None: + selected_household_ids = tuple(household_frame["household_id"].tolist()) + inclusion_probability = 1.0 else: - selected_household_ids = _validate_selected_household_ids( + expected = _expected_spi_households(expected_spi_households) + source_households = _distinct_source_households(household_frame) + inclusion_probability = min(1.0, expected / source_households) + selected_household_ids = _sample_spi_household_ids( household_frame, - selected_household_ids, + inclusion_probability=inclusion_probability, + seed=seed, ) selected_household_set = set(selected_household_ids) selected_person = person_frame[ @@ -376,6 +428,9 @@ def create_uk_spi_support_tables( household=pd.concat([base_household, spi_household], ignore_index=True), id_multiplier=copy_id_table.radix(SPI_SUPPORT_COPY_ID_FIELD), spi_household_ids=selected_household_ids, + expected_spi_households=expected, + spi_source_households=source_households, + spi_inclusion_probability=inclusion_probability, ) _validate_uk_support_outputs(result.person, result.benunit, result.household) return result @@ -483,7 +538,7 @@ def replace_uk_spi_support_tables( person=surviving_people, benunit=surviving_benunits, household=surviving_households, - spi_household_count=None, + expected_spi_households=None, seed=seed, source_year=source_year, selected_household_ids=replacement_source_ids, @@ -513,7 +568,7 @@ def build_uk_spi_support_channel( benunit: pd.DataFrame, household: pd.DataFrame, *, - spi_household_count: int = DEFAULT_SPI_SUPPORT_HOUSEHOLDS, + expected_spi_households: float | None = DEFAULT_SPI_SUPPORT_HOUSEHOLDS, seed: int = 42, source_year: int | None = None, spi_prior_mass_share: float = DEFAULT_SPI_PRIOR_MASS_SHARE, @@ -528,13 +583,23 @@ def build_uk_spi_support_channel( ) -> UKSPISupportResult: """Stack the E7 SPI support channel and allocate prior mass by stratum. + The channel is the keyed Bernoulli share of + :func:`create_uk_spi_support_tables`. A zero-weight declaration given as + a mapping may bound its stratum by ``maximum_zero_weight_rows_rule`` + instead of a fixed maximum; the rule is resolved after the draw, over the + ``n`` household rows entering the stage, to ``min(n, ceil(n x (pi + + band)))`` (:func:`spi_share_gate_maximum`), and the gate then runs as for + any declaration. + With ``pension_age_share`` the strata also split households with a member at or over ``state_pension_age`` from the rest, and the SPI channel takes that share of the pension-age strata's mass (``spi_prior_mass_share`` of the others). Every adult in an SPI household carries a taxpayer's income, and the channel's households are drawn unweighted with one weight per region, so without the split the channel holds more pension-age mass than - the FRS design gives pension-age households (microcosm#1069). + the FRS design gives pension-age households (microcosm#1069). Where the + frame carries geographic clones, the strata split by clone as well + (:func:`_allocate_spi_prior_mass`). """ if not isinstance(input_weight_kind, WeightKind): @@ -564,16 +629,23 @@ def build_uk_spi_support_channel( f"{SPI_PENSION_AGE_STRATUM_COLUMN} is derived by the allocation; " "do not pass it as a stratum column." ) + # The clone strata are checked on the incoming households, so a refusal + # counts them rather than them and their copies. + _geographic_clone_strata(household) stacked = create_uk_spi_support_tables( person=person, benunit=benunit, household=household, - spi_household_count=spi_household_count, + expected_spi_households=expected_spi_households, seed=seed, source_year=source_year, ) - declarations = _coerce_zero_weight_declarations(zero_weight_declarations) + declarations = _coerce_zero_weight_declarations( + zero_weight_declarations, + inclusion_probability=stacked.spi_inclusion_probability, + household_rows=len(household), + ) if declarations: gate = uk_zero_weight_strata_gate( stacked.household, @@ -581,8 +653,7 @@ def build_uk_spi_support_channel( ) if not gate.passed: raise ValueError( - "UK SPI support pre-clone zero-weight gate failed: " - f"{list(gate.failures)}." + f"UK SPI support zero-weight gate failed: {list(gate.failures)}." ) if pension_age_share is None: return _allocate_spi_prior_mass( @@ -706,13 +777,31 @@ def fill_support_channel_from_source( def _coerce_zero_weight_declarations( declarations: Sequence[UKZeroWeightStratumDeclaration] | Sequence[Mapping[str, Any]], + *, + inclusion_probability: float | None, + household_rows: int, ) -> tuple[UKZeroWeightStratumDeclaration, ...]: + """The declarations as the gate takes them, each rule resolved to a maximum.""" + materialized: list[UKZeroWeightStratumDeclaration] = [] for declaration in declarations: if isinstance(declaration, UKZeroWeightStratumDeclaration): materialized.append(declaration) elif isinstance(declaration, Mapping): - materialized.append(UKZeroWeightStratumDeclaration(**dict(declaration))) + fields = dict(declaration) + rule = fields.pop("maximum_zero_weight_rows_rule", None) + if rule is not None: + if "maximum_zero_weight_rows" in fields: + raise ValueError( + f"UK zero-weight stratum {fields.get('name')!r} declares " + "both a maximum and a maximum rule." + ) + fields["maximum_zero_weight_rows"] = spi_share_gate_maximum( + rule, + inclusion_probability=inclusion_probability, + household_rows=household_rows, + ) + materialized.append(UKZeroWeightStratumDeclaration(**fields)) else: raise TypeError( "zero_weight_declarations must contain " @@ -721,6 +810,80 @@ def _coerce_zero_weight_declarations( return tuple(materialized) +def spi_share_gate_maximum( + rule: Mapping[str, Any], + *, + inclusion_probability: float | None, + household_rows: int, +) -> int: + """The SPI share gate's maximum: ``min(n, ceil(n x (pi + band(pi, n))))``. + + ``n`` is the household rows entering the stage and ``pi`` the share's + inclusion probability. ``rule`` is ``{"rule": "binomial_share_band", + "sigma": s}``: the band is the repo's binomial band at ``s`` standard + errors (:func:`~microcosm.build.stochastic_assignment. + binomial_share_tolerance`), so a share drawn at ``pi`` passes unless its + count lies more than ``s`` standard errors above ``n x pi`` (microcosm#931). + """ + + if ( + not isinstance(rule, Mapping) + or set(rule) != {"rule", "sigma"} + or rule["rule"] != SPI_SHARE_GATE_RULE + ): + raise ValueError( + "A zero-weight maximum rule must be {'rule': " + f"{SPI_SHARE_GATE_RULE!r}, 'sigma': }}." + ) + sigma = rule["sigma"] + if ( + isinstance(sigma, bool) + or not isinstance(sigma, int | float) + or not math.isfinite(sigma) + or sigma <= 0 + ): + raise ValueError("A zero-weight maximum rule's sigma must be positive.") + if inclusion_probability is None: + raise ValueError( + "A zero-weight maximum rule needs the SPI share's inclusion " + "probability; named copies are not a share." + ) + rows = int(household_rows) + probability = float(inclusion_probability) + band = binomial_share_tolerance(target=probability, rows=rows, sigma=float(sigma)) + return min(rows, math.ceil(rows * (probability + band))) + + +def spi_share_inclusion( + household_ids: Sequence[object] | np.ndarray, + *, + inclusion_probability: float, + seed: int, +) -> np.ndarray: + """Which households the SPI share copies: ``True`` where one is drawn. + + A household is drawn when ``inclusion_probability`` is one, or when its + uniform, keyed on its own integer id under ``seed`` and + :data:`SPI_SUPPORT_INCLUSION_SALT`, falls below it. Each household's draw + reads its id, the seed, the salt and the probability alone, so permuting + the rows, or adding other households at the same probability, leaves it + unchanged; a geographic clone's id carries its clone, so each clone draws + on its own (microcosm#931). At one no uniform is read, since an integer + over ``2**64`` can round up to one. + """ + + keys = integer_record_keys(household_ids, label=UK_SPI_SUPPORT_STAGE_NAME) + probability = float(inclusion_probability) + if not math.isfinite(probability) or not 0.0 < probability <= 1.0: + raise ValueError("The SPI share's inclusion probability must lie in (0, 1].") + if probability >= 1.0: + return np.ones(keys.size, dtype=bool) + uniforms = stable_identity_uniforms( + keys, seed=seed, salt=SPI_SUPPORT_INCLUSION_SALT + ) + return uniforms < probability + + def support_channel_column(entity: str) -> str: """Return the entity-prefixed support-channel column name.""" @@ -785,31 +948,49 @@ def _prepare_household_lineage( return frame +def _expected_spi_households(value: object) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, int | float | np.integer | np.floating) + or not math.isfinite(float(value)) + or float(value) <= 0.0 + ): + raise ValueError("expected_spi_households must be a positive number or None.") + return float(value) + + +def _distinct_source_households(household: pd.DataFrame) -> int: + """The share's denominator: the distinct true sources of the households.""" + + sources = household["source_household_id"] + unsourced = int(sources.isna().sum()) + if unsourced: + raise ValueError( + f"{unsourced} household row(s) carry no source_household_id; the SPI " + "share counts distinct source households." + ) + count = int(sources.nunique()) + if count == 0: + raise ValueError("The SPI share needs at least one source household.") + return count + + def _sample_spi_household_ids( household: pd.DataFrame, *, - spi_household_count: int | None, + inclusion_probability: float, seed: int, ) -> tuple[Any, ...]: - if spi_household_count is None: - return tuple(household["household_id"].tolist()) - if not isinstance(spi_household_count, int) or spi_household_count <= 0: - raise ValueError("spi_household_count must be a positive integer or None.") - if spi_household_count > len(household): - raise ValueError( - "spi_household_count cannot exceed the number of household rows." - ) + """The households the share copies, in frame order.""" + if not isinstance(seed, int): raise ValueError("seed must be an integer.") - - rng = np.random.default_rng(seed) - selected_positions = set( - rng.choice(len(household), size=spi_household_count, replace=False).tolist() + selected = spi_share_inclusion( + household["household_id"].to_numpy(), + inclusion_probability=inclusion_probability, + seed=seed, ) - selected = household.iloc[ - [index for index in range(len(household)) if index in selected_positions] - ] - return tuple(selected["household_id"].tolist()) + return tuple(household.loc[selected, "household_id"].tolist()) def _validate_selected_household_ids( @@ -909,6 +1090,18 @@ def _allocate_spi_prior_mass( strata_columns: tuple[str, ...], pension_age_share: float | None = None, ) -> UKSPISupportResult: + """Give the SPI channel its share of every stratum's base mass. + + Where the household table carries ``household_clone_index`` (the + clone-first spine's geographic clones, microcosm#931), each clone is its + own stratum within ``strata_columns``, so every clone keeps its own mass + and a clone's SPI weights depend on its own base households and copies + alone; the stratum is derived for the allocation only and never written + to the frame. A household table without the column, as on every national + spine today, allocates exactly as before. A positive-mass stratum with no + SPI copy is refused, by count. + """ + household = result.household.copy() channels = household[support_channel_column("household")] base_mask = channels.eq(BASE_FRS_SUPPORT_CHANNEL).to_numpy() @@ -923,11 +1116,21 @@ def _allocate_spi_prior_mass( ).to_numpy(dtype=float, na_value=np.nan) old_total = float(pre_weights.sum()) _require_columns(household, strata_columns, label="rebuilt household") + if SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN in strata_columns: + raise ValueError( + f"{SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN} is derived by the allocation; " + "do not pass it as a stratum column." + ) allocation = household[list(strata_columns)].copy() + group_columns = list(strata_columns) + clone_strata = _geographic_clone_strata(household) + if clone_strata is not None: + allocation[SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN] = clone_strata + group_columns.append(SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN) allocation["_base_mass"] = np.where(base_mask, pre_weights, 0.0) allocation["_spi_count"] = spi_mask.astype(np.int64) grouped = allocation.groupby( - list(strata_columns), + group_columns, sort=False, dropna=False, ) @@ -937,15 +1140,13 @@ def _allocate_spi_prior_mass( base_mask & (stratum_base_mass > 0.0) & (stratum_spi_count == 0) ) if unrepresented_base.any(): - examples = ( - household.loc[unrepresented_base, list(strata_columns)] - .drop_duplicates() - .head(5) - .to_dict(orient="records") - ) + stratum = grouped.ngroup().to_numpy() + positive = base_mask & (stratum_base_mass > 0.0) raise ValueError( "Every positive-mass UK base stratum must have rebuilt SPI support; " - f"missing stratum example(s): {examples}." + f"{np.unique(stratum[unrepresented_base]).size} of the " + f"{np.unique(stratum[positive]).size} positive-mass strata have no " + "SPI copy." ) invalid_spi = spi_mask & ( ~np.isfinite(stratum_base_mass) @@ -1024,9 +1225,46 @@ def _allocate_spi_prior_mass( spi_prior_mass_share=spi_prior_mass_share, pension_age_spi_prior_mass_share=pension_age_share, pension_age_allocation=pension_age_allocation, + expected_spi_households=result.expected_spi_households, + spi_source_households=result.spi_source_households, + spi_inclusion_probability=result.spi_inclusion_probability, ) +def _geographic_clone_strata(household: pd.DataFrame) -> np.ndarray | None: + """Each household's geographic clone, or None without the index column. + + Refuses a non-integer column, a household without an index and a + negative index, each by count or dtype, never by id or value. + """ + + column = SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN + if column not in household.columns: + return None + values = household[column] + if pd.api.types.is_bool_dtype(values.dtype) or not ( + pd.api.types.is_integer_dtype(values.dtype) + ): + raise ValueError( + "The SPI prior allocation stratifies geographic clones on an integer " + f"{column!r} column; it has dtype {values.dtype}." + ) + unindexed = int(values.isna().sum()) + if unindexed: + raise ValueError( + f"{unindexed} household row(s) carry no {column!r}; the SPI prior " + "allocation stratifies each geographic clone." + ) + indices = values.to_numpy(dtype=np.int64) + negative = int((indices < 0).sum()) + if negative: + raise ValueError( + f"{negative} household row(s) carry a negative {column!r}; the SPI " + "prior allocation stratifies each geographic clone." + ) + return indices + + def _importance_weights_with_exact_total( values: np.ndarray, target: float, @@ -1332,6 +1570,8 @@ def _require_entity(entity: str) -> str: "SPI_HMRC_UNEMPLOYMENT_BENEFIT_INCOME_COLUMN", "SPI_INCOME_IMPUTATION_COLUMNS", "SPI_INCOME_QRF_OUTPUT_COLUMNS", + "SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN", + "SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN", "SPI_LOSSES_BROUGHT_FORWARD_DRAW_COLUMN", "SPI_LOSS_DRAW_COLUMNS", "SPI_PENSION_AGE_STRATUM_COLUMN", @@ -1339,14 +1579,18 @@ def _require_entity(entity: str) -> str: "SPI_PROFIT_BEFORE_LOSSES_DRAW_COLUMN", "SPI_PROPERTY_FINANCE_COSTS_COLUMN", "SPI_REPLACEMENT_STRATA_COLUMNS", + "SPI_SHARE_GATE_RULE", "SPI_SUPPORT_COPY_ID_FIELD", "SPI_SUPPORT_ID_REMAPPING", + "SPI_SUPPORT_INCLUSION_SALT", "UKSPISupportResult", "UK_SPI_SUPPORT_STAGE_NAME", "build_uk_spi_support_channel", "create_uk_spi_support_tables", "fill_support_channel_from_source", "replace_uk_spi_support_tables", + "spi_share_gate_maximum", + "spi_share_inclusion", "support_channel_column", "support_clone_index_column", "support_source_id_column", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index 4809d5c549..965fce8837 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -8,6 +8,7 @@ import numpy as np from microcosm.build.gates import GateResult +from microcosm.build.stochastic_assignment import binomial_share_tolerance from microcosm.build.uk_runtime.receipt_counts import ( UK_BELOW_TEN_BAND, UK_DISCLOSURE_FLOOR, @@ -2809,16 +2810,14 @@ def _latent_attribute_realization_gate( def latent_attribute_tolerance(*, target: float, rows: int) -> float: - """Binomial band for a share realized over ``rows`` identity-keyed draws.""" - - if rows <= 0: - return 1.0 - return min( - 1.0, - max( - 1.0 / rows, - LATENT_ATTRIBUTE_SIGMA * math.sqrt(target * (1.0 - target) / rows), - ), + """Binomial band for a share realized over ``rows`` identity-keyed draws. + + The shared band (:func:`~microcosm.build.stochastic_assignment. + binomial_share_tolerance`) at :data:`LATENT_ATTRIBUTE_SIGMA`. + """ + + return binomial_share_tolerance( + target=target, rows=rows, sigma=LATENT_ATTRIBUTE_SIGMA ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py index 7459ec1277..1c34c4aab1 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py @@ -1,5 +1,7 @@ from __future__ import annotations +import math + import numpy as np import pytest @@ -269,3 +271,44 @@ def test_a_weightless_stratum_keeps_its_draws_and_misalignment_is_refused() -> N weight_balanced_uniforms(np.zeros(2), np.zeros(3), np.zeros(3)) with pytest.raises(ValueError, match="finite"): weight_balanced_uniforms(np.zeros(2), np.zeros(2), np.array([1.0, np.nan])) + + +def test_binomial_share_tolerance_is_sigma_standard_errors_floored_and_capped() -> None: + from microcosm.build.stochastic_assignment import binomial_share_tolerance + + assert binomial_share_tolerance(target=0.3, rows=1_000, sigma=4.0) == ( + 4.0 * (0.3 * 0.7 / 1_000) ** 0.5 + ) + assert binomial_share_tolerance(target=0.3, rows=1_000, sigma=2.0) == ( + 2.0 * (0.3 * 0.7 / 1_000) ** 0.5 + ) + # Floored at one row's worth, capped at one, and one without rows. + assert binomial_share_tolerance(target=1.0, rows=50, sigma=4.0) == 1.0 / 50 + assert binomial_share_tolerance(target=0.5, rows=1, sigma=4.0) == 1.0 + assert binomial_share_tolerance(target=0.5, rows=0, sigma=4.0) == 1.0 + + +def test_the_latent_attribute_band_is_the_shared_band_at_four_sigma() -> None: + # The stage-health name stays an alias of the moved band (microcosm#931). + from microcosm.build.stochastic_assignment import binomial_share_tolerance + from microcosm.build.uk_runtime.stage_health import ( + LATENT_ATTRIBUTE_SIGMA, + latent_attribute_tolerance, + ) + + assert LATENT_ATTRIBUTE_SIGMA == 4.0 + for target in (0.0, 0.01, 0.3, 0.5, 0.97, 1.0): + for rows in (0, 1, 7, 250, 40_000): + # The band as stage_health computed it before the move. + before = ( + 1.0 + if rows <= 0 + else min( + 1.0, + max(1.0 / rows, 4.0 * math.sqrt(target * (1.0 - target) / rows)), + ) + ) + assert latent_attribute_tolerance(target=target, rows=rows) == before + assert binomial_share_tolerance(target=target, rows=rows, sigma=4.0) == ( + before + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py index 92964537e6..beffddd324 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -405,7 +405,7 @@ def _spi_support_on_clone_first_tables(base: IdTables, n_clones: int): household=tables.household[ ["household_id", "source_household_id", "household_clone_index"] ].assign(household_weight=1.0), - spi_household_count=None, + expected_spi_households=None, selected_household_ids=tuple( source + clone * GEOGRAPHIC_RADIX for clone in range(n_clones) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index ff1154876c..fbd8dfcf1c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -1091,15 +1091,19 @@ def source_stage( "kind": "stack_zero_weight_donors", "count": 10000, "seed": 42, - "draw": "uniform_without_replacement", + "salt": "spi_support_channel:inclusion", + "draw": "keyed_bernoulli_share", }, { "kind": "gate_zero_weight_strata", "declarations": [ { - "name": "e7_spi_synthetic_preclone", + "name": "e7_spi_synthetic_share", "selector": {"household_is_spi_synthetic": True}, - "maximum_zero_weight_rows": 10000, + "maximum_zero_weight_rows_rule": { + "rule": "binomial_share_band", + "sigma": 4, + }, "reason": "synthetic driver fixture", } ], @@ -1904,7 +1908,7 @@ def test_birth_months_are_spread_at_the_frs_weights_and_kept_by_copies( person=tables_with_weights["person"], benunit=tables_with_weights["benunit"], household=tables_with_weights["household"], - spi_household_count=None, + expected_spi_households=None, ) copies = support.person[ support.person[support_channel_column("person")] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_income_source_manifest.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_income_source_manifest.py index a1bc0fdc97..81e1f8f640 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_income_source_manifest.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_hmrc_income_source_manifest.py @@ -119,6 +119,42 @@ def test_official_hmrc_sources_and_sampling_weights_remain_bound(): 0, "count drifted", ), + # The keyed Bernoulli share and its gate (microcosm#931). + ( + "spi_support_channel", + "stack_zero_weight_donors", + "draw", + "uniform_without_replacement", + "keyed Bernoulli share draw", + ), + ( + "spi_support_channel", + "stack_zero_weight_donors", + "salt", + "spi_support_channel", + "salt drifted", + ), + ( + "spi_support_channel", + "gate_zero_weight_strata", + "declarations", + [ + { + "name": "e7_spi_synthetic_share", + "selector": {"household_is_spi_synthetic": True}, + "maximum_zero_weight_rows": 10000, + "reason": "an exact count", + } + ], + "share gate declaration drifted", + ), + ( + "spi_support_channel", + "allocate_zero_weight_prior_mass", + "geographic_clone_stratum", + "clone_index", + "by geographic clone", + ), (INCOME, "strict_read_private_table", "weight", "uniform", "strict.weight"), ( INCOME, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py index e8495c452d..5ce6d4a1d0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py @@ -1154,6 +1154,12 @@ def test_e7_declared_seed_lockstep(self) -> None: stages = {stage.stage: stage for stage in spec.sources.stages} assert stages["spi_support_channel"].operations[0].parameters["seed"] == 42 + # The share's inclusion uniforms are keyed under their own salt + # (microcosm#931). + assert ( + stages["spi_support_channel"].operations[0].parameters["salt"] + == "spi_support_channel:inclusion" + ) assert stages["hmrc_spi_income_spine"].operations[3].parameters["seed"] == 42 # The reserved carriers' resample draws at stage seed + 2 (PolicyEngine/chronicle#280 lane). assert stages["hmrc_spi_income_spine"].operations[4].parameters["seed"] == 44 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index 300e3fd307..eee420016d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -197,11 +197,15 @@ def _support_frame( "region": np.where(ids % 3 == 0, "SOUTH_EAST", "LONDON"), } ) + # The SPI copies are a keyed Bernoulli share at 10 / n (microcosm#931), so + # their count is binomial. At seed 42 both regions keep copies at the 60 + # and 200 households these tests build (11 copies, 2 in SOUTH_EAST, at + # 60; 13, 5 in SOUTH_EAST, at 200), so every funding stratum has SPI mass. support = build_uk_spi_support_channel( person, benunit, household, - spi_household_count=10, + expected_spi_households=10, seed=42, source_year=2024, zero_weight_declarations=(), @@ -493,11 +497,16 @@ def test_stack_follows_weight_times_propensity_not_the_sample() -> None: # seats follow household weight: the heavier half of the frame is seated # more often than its half share of the rows across many seeds. frame = _support_frame(200) + household = frame.table("household") weight = pd.Series( frame.weights_for("household").values, - index=frame.table("household")["household_id"].to_numpy(), + index=household["household_id"].to_numpy(), ) - heavy = weight > weight[weight > 0.0].median() + # The heavier half of the FRS base, the households seats are drawn from: + # a median over every row would count the SPI copies, whose number is + # binomial (microcosm#931), and leave the heavy set short of half. + base = household[support_channel_column("household")].eq(BASE_FRS_SUPPORT_CHANNEL) + heavy = weight > weight[base.to_numpy()].median() heavy_seats = seats = 0 propensity = _propensity() for seed in range(40): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py index f44dd1200a..11639a6bbf 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py @@ -30,6 +30,7 @@ from microcosm.build.uk_runtime.spi_spine import ( EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN, SPI_SPINE_PENSION_AGE_PRIOR_MASS_SHARE, + SPI_SPINE_SUPPORT_DRAW, UKFRSHMRCSpineLeavesStageTransform, UKSPIIncomeSpineStageTransform, UKSPISupportChannelStageTransform, @@ -46,6 +47,7 @@ ) from microcosm.build.uk_runtime.terminal_gates import UKZeroWeightStratumDeclaration from microcosm.frame import WeightKind +from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame def _base_frame(*, time_period: str = "2023") -> object: @@ -253,15 +255,19 @@ def _support_stage() -> SourceStageSpec: "kind": "stack_zero_weight_donors", "count": 10000, "seed": 42, - "draw": "uniform_without_replacement", + "salt": "spi_support_channel:inclusion", + "draw": SPI_SPINE_SUPPORT_DRAW, }, { "kind": "gate_zero_weight_strata", "declarations": [ { - "name": "e7_spi_synthetic_preclone", + "name": "e7_spi_synthetic_share", "selector": {HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True}, - "maximum_zero_weight_rows": 10000, + "maximum_zero_weight_rows_rule": { + "rule": "binomial_share_band", + "sigma": 4, + }, "reason": "synthetic fixture", } ], @@ -297,7 +303,7 @@ def test_support_transform_refuses_manifest_code_drift() -> None: UKSPISupportChannelStageTransform(stage=stage)(_base_frame()) -def test_preclone_support_gate_passes_at_limit_and_fails_above() -> None: +def test_share_support_gate_fails_above_its_maximum() -> None: household = pd.DataFrame( { "household_id": [1, 2], @@ -306,17 +312,18 @@ def test_preclone_support_gate_passes_at_limit_and_fails_above() -> None: } ) declaration = UKZeroWeightStratumDeclaration( - name="e7_spi_synthetic_preclone", + name="e7_spi_synthetic_share", selector={HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True}, maximum_zero_weight_rows=1, reason="synthetic fixture", ) + # An expected count of 2 over 2 households copies both. with pytest.raises(ValueError, match="exceed"): build_uk_spi_support_channel( person=_base_frame().table("person"), benunit=_base_frame().table("benunit"), household=household, - spi_household_count=2, + expected_spi_households=2, zero_weight_declarations=(declaration,), ) @@ -516,10 +523,10 @@ def test_spi_income_zero_initializes_frs_charity_and_keeps_frs_dividends( "region": ["LONDON", "SCOTLAND"], } ), - spi_household_count=2, + expected_spi_households=2, zero_weight_declarations=( UKZeroWeightStratumDeclaration( - name="e7_spi_synthetic_preclone", + name="e7_spi_synthetic_share", selector={HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True}, maximum_zero_weight_rows=2, reason="synthetic fixture", @@ -601,10 +608,10 @@ def test_reviewed_absent_incapacity_signal_raises(tmp_path: Path) -> None: "region": ["LONDON", "SCOTLAND"], } ), - spi_household_count=2, + expected_spi_households=2, zero_weight_declarations=( UKZeroWeightStratumDeclaration( - name="e7_spi_synthetic_preclone", + name="e7_spi_synthetic_share", selector={HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True}, maximum_zero_weight_rows=2, reason="synthetic fixture", @@ -933,10 +940,61 @@ def test_support_stage_parameters_accept_the_committed_manifest() -> None: assert count == 10000 assert share == 0.5 assert strata == ("region",) - assert len(declarations) == 1 + # The one share gate, bounded by the four-sigma binomial band rather than + # a fixed maximum (microcosm#931). + (declaration,) = declarations + assert declaration["name"] == "e7_spi_synthetic_share" + assert declaration["selector"] == {HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True} + assert declaration["maximum_zero_weight_rows_rule"] == { + "rule": "binomial_share_band", + "sigma": 4.0, + } + assert "maximum_zero_weight_rows" not in declaration assert pension_age_share == SPI_SPINE_PENSION_AGE_PRIOR_MASS_SHARE +@pytest.mark.parametrize( + ("kind", "changes", "message"), + [ + ( + "stack_zero_weight_donors", + {"draw": "uniform_without_replacement"}, + "keyed Bernoulli share draw", + ), + ("stack_zero_weight_donors", {"salt": "spi_support_channel"}, "salt drifted"), + ("stack_zero_weight_donors", {"salt": None}, "salt drifted"), + ( + "allocate_zero_weight_prior_mass", + {"geographic_clone_stratum": "clone_index"}, + "by geographic clone", + ), + ( + "allocate_zero_weight_prior_mass", + {"geographic_clone_stratum": None}, + "by geographic clone", + ), + ], +) +def test_support_stage_parameters_refuse_draw_salt_and_clone_stratum_drift( + kind: str, changes: dict, message: str +) -> None: + committed = _committed_stage("spi_support_channel") + operations = [] + for operation in committed.operations: + payload = {"kind": operation.kind, **dict(operation.parameters)} + if operation.kind == kind: + payload.update(changes) + payload = { + key: value for key, value in payload.items() if value is not None + } + operations.append(payload) + stage = SourceStageSpec.from_mapping( + {**committed.__dict__, "operations": operations} + ) + with pytest.raises(ValueError, match=message): + _support_stage_parameters(stage, seed=42) + + @pytest.mark.parametrize( ("changes", "message"), [ @@ -996,15 +1054,30 @@ def _stub_builder(*_args, **_kwargs) -> UKSPISupportResult: transform(frame) -def test_support_stage_parameters_refuse_gate_declaration_drift() -> None: +@pytest.mark.parametrize( + "changes", + [ + {"name": "e7_spi_synthetic_preclone"}, + {"selector": {HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: False}}, + {"maximum_zero_weight_rows": 20000}, + {"maximum_zero_weight_rows_rule": None, "maximum_zero_weight_rows": 10000}, + {"maximum_zero_weight_rows_rule": {"rule": "normal_band", "sigma": 4}}, + {"maximum_zero_weight_rows_rule": {"rule": "binomial_share_band", "sigma": 3}}, + {"maximum_zero_weight_rows_rule": {"rule": "binomial_share_band"}}, + {"reason": " "}, + ], +) +def test_support_stage_parameters_refuse_gate_declaration_drift(changes: dict) -> None: committed = _committed_stage("spi_support_channel") gate = next( operation for operation in committed.operations if operation.kind == "gate_zero_weight_strata" ) - declaration = dict(gate.parameters["declarations"][0]) - declaration["maximum_zero_weight_rows"] = 20000 + declaration = {**dict(gate.parameters["declarations"][0]), **changes} + declaration = { + key: value for key, value in declaration.items() if value is not None + } stage = _with_mutated_operation( committed, "gate_zero_weight_strata", declarations=[declaration] ) @@ -1049,3 +1122,143 @@ def test_readers_of_the_support_record_take_the_declared_allocation_reason() -> manifest = load_uk_release_input_coverage_manifest() family = manifest.family_coverage["hmrc_spi_income"] assert family["required_mass_change_reason"] == reason + + +# The keyed Bernoulli share at the stage (microcosm#931). Every frame below is +# synthetic. + +_SHARE_REGIONS = np.asarray(["LONDON", "WALES", "SCOTLAND", "NORTH_EAST"]) + + +def _share_frame(n_households: int = 400) -> object: + """One-person households over four regions, some at State Pension age.""" + + rng = np.random.default_rng(11) + ids = np.arange(1, n_households + 1, dtype="int64") + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": ids * 1_000 + 1, + "person_household_id": ids, + "person_benunit_id": ids * 100 + 1, + "age": 25 + ids % 60, + } + ), + benunit=pd.DataFrame({"benunit_id": ids * 100 + 1}), + household=pd.DataFrame( + { + "household_id": ids, + "household_weight": rng.uniform(50.0, 500.0, n_households), + "region": _SHARE_REGIONS[ids % 4], + } + ), + time_period="2024", + weight_kind=WeightKind.DESIGN, + ) + + +def _share_stage(frame, *, sample_fraction: float = 0.01): + transform = UKSPISupportChannelStageTransform( + stage=_committed_stage("spi_support_channel"), + sample_fraction=sample_fraction, + state_pension_age=66, + ) + out = transform(frame) + return out, transform.checkpoint_metadata()["evidence"] + + +def test_the_stage_draws_the_declared_count_times_the_sample_fraction() -> None: + from microcosm.build.stochastic_assignment import binomial_share_tolerance + + out, evidence = _share_stage(_share_frame()) + copies = int( + out.table("household")["household_support_channel"] + .eq(SPI_SYNTHETIC_SUPPORT_CHANNEL) + .sum() + ) + + # 10,000 x 0.01 = 100 expected copies over 400 sources: a share of 1/4. + assert evidence["expected_spi_households"] == 100.0 + assert evidence["inclusion_probability"] == 0.25 + assert evidence["source_households"] == 400 + assert evidence["spi_households"] == copies + assert abs(copies - 100) <= 400 * binomial_share_tolerance( + target=0.25, rows=400, sigma=4.0 + ) + # The gate is recorded by its rule, not the maximum it resolved to. + assert evidence["zero_weight_gate"] == { + "name": "e7_spi_synthetic_share", + "rule": "binomial_share_band", + "sigma": 4.0, + } + # The receipt rounds the source count and floors the share. + _, rounded = _share_stage(_share_frame(397), sample_fraction=0.0101) + assert rounded["source_households"] == 400 + assert rounded["inclusion_probability"] == 0.25 + + +def test_an_expected_count_at_the_source_count_copies_every_household() -> None: + # As in the H2 fixture: 10,000 x (135 / 10,000) is 135.0 in float, so the + # share is one and every household is copied, as the count did before. + fraction = 135 / 10_000 + assert 10_000 * fraction == 135.0 + out, evidence = _share_stage(_share_frame(135), sample_fraction=fraction) + household = out.table("household") + spi = household["household_support_channel"].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL) + + assert evidence["inclusion_probability"] == 1.0 + assert evidence["spi_households"] == 135 + assert household.loc[spi, "household_source_id"].tolist() == list(range(1, 136)) + + +@pytest.mark.parametrize("clone_count", [3, 7]) +def test_clone_zero_of_a_cloned_spine_is_the_one_clone_share(clone_count: int) -> None: + from microcosm.build.stochastic_assignment import binomial_share_tolerance + + one, one_evidence = _share_stage(clone_first_frame(_share_frame(), 1)) + many, many_evidence = _share_stage(clone_first_frame(_share_frame(), clone_count)) + one_household = one.table("household") + many_household = many.table("household") + + def copied(household, clone: int) -> set[int]: + rows = household["household_clone_index"].eq(clone) & household[ + "household_support_channel" + ].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL) + return set(household.loc[rows, "source_household_id"].tolist()) + + # The share divides by the sources, not the rows, so it is the same at + # every clone count, and clone 0, whose ids are the one-clone ids, draws + # exactly the one-clone copies. The other clones draw on their own ids. + assert many_evidence["inclusion_probability"] == 0.25 + assert many_evidence["source_households"] == one_evidence["source_households"] + reference = copied(one_household, 0) + assert copied(many_household, 0) == reference + for clone in range(1, clone_count): + assert copied(many_household, clone) != reference + rows = clone_count * 400 + copies = many_evidence["spi_households"] + assert abs(copies - rows * 0.25) <= rows * binomial_share_tolerance( + target=0.25, rows=rows, sigma=4.0 + ) + + # Nested weights: K x every clone-0 row is the one-clone row. The + # exact-total correction lands on clone 0's rows, so to rtol 1e-12, not + # bit for bit. + one_weights = pd.Series( + np.asarray(one.weights_for("household").values), + index=one_household["household_id"].to_numpy(), + ) + many_weights = pd.Series( + np.asarray(many.weights_for("household").values), + index=many_household["household_id"].to_numpy(), + ) + clone_zero = many_household.loc[ + many_household["household_clone_index"].eq(0), "household_id" + ].to_numpy() + assert set(clone_zero) == set(one_weights.index) + np.testing.assert_allclose( + clone_count * many_weights.loc[clone_zero].to_numpy(), + one_weights.loc[clone_zero].to_numpy(), + rtol=1e-12, + atol=0.0, + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py index 29b1b8c9e3..76234eabdc 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py @@ -1,9 +1,12 @@ from __future__ import annotations +import math + import numpy as np import pandas as pd import pytest +from microcosm.build.stochastic_assignment import binomial_share_tolerance from microcosm.build.uk_runtime import ( BASE_FRS_SUPPORT_CHANNEL, DEFAULT_SPI_PRIOR_MASS_SHARE, @@ -23,7 +26,10 @@ support_clone_index_column, support_source_id_column, ) -from microcosm.build.uk_runtime.terminal_gates import UKZeroWeightStratumDeclaration +from microcosm.build.uk_runtime.terminal_gates import ( + UKZeroWeightStratumDeclaration, + uk_zero_weight_strata_gate, +) from microcosm.frame import MassChangeRecord, WeightKind from test_support.microcosm_build.uk_clone_first_frames import ( assert_copy_ids_follow_lineage, @@ -65,7 +71,7 @@ def test_spi_support_creates_zero_weight_copy_with_lineage_and_remapped_ids() -> person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=None, + expected_spi_households=None, source_year=2023, ) @@ -181,7 +187,7 @@ def test_spi_copies_take_their_roots_ids_plus_the_declared_radix() -> None: "household": household_frame(), } result = create_uk_spi_support_tables( - **before, spi_household_count=None, source_year=2023 + **before, expected_spi_households=None, source_year=2023 ) after = { "person": result.person, @@ -214,7 +220,7 @@ def test_spi_support_refuses_incoming_ids_that_reach_its_copy_id_field() -> None person=person, benunit=benunit_frame(), household=household, - spi_household_count=None, + expected_spi_households=None, source_year=2023, ) # A count, never the id. @@ -222,12 +228,14 @@ def test_spi_support_refuses_incoming_ids_that_reach_its_copy_id_field() -> None def test_spi_support_can_subsample_without_rescaling_or_reordering_base() -> None: + # An expected count of 2 over 3 source households draws each at 2/3; + # seed 0 keys households 1 and 2 in and 3 out (microcosm#931). result = create_uk_spi_support_tables( person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=2, - seed=1, + expected_spi_households=2, + seed=0, ) assert result.household.head(3)["household_id"].tolist() == [1, 2, 3] @@ -236,11 +244,14 @@ def test_spi_support_can_subsample_without_rescaling_or_reordering_base() -> Non result.household[support_channel_column("household")] == SPI_SYNTHETIC_SUPPORT_CHANNEL ] - assert len(spi_households) == 2 + assert result.spi_household_ids == (1, 2) assert spi_households["household_weight"].eq(0).all() assert tuple(spi_households[support_source_id_column("household")]) == ( result.spi_household_ids ) + assert result.expected_spi_households == 2.0 + assert result.spi_source_households == 3 + assert result.spi_inclusion_probability == 2 / 3 def test_spi_fill_only_updates_spi_channel_and_can_initialize_new_columns() -> None: @@ -248,7 +259,7 @@ def test_spi_fill_only_updates_spi_channel_and_can_initialize_new_columns() -> N person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=None, + expected_spi_households=None, ) donor = pd.DataFrame( { @@ -393,11 +404,12 @@ def test_build_spi_support_channel_fresh_stack_allocates_region_mass() -> None: reason="prior raw FRS calibration", ) + # An expected count of 3 over 3 source households copies every one. result = build_uk_spi_support_channel( person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=3, + expected_spi_households=3, seed=42, source_year=2023, strata_columns=("region",), @@ -405,7 +417,7 @@ def test_build_spi_support_channel_fresh_stack_allocates_region_mass() -> None: mass_log=(prior_record,), zero_weight_declarations=( UKZeroWeightStratumDeclaration( - name="e7_spi_synthetic_preclone", + name="e7_spi_synthetic_share", selector={HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True}, maximum_zero_weight_rows=3, reason="synthetic test declaration", @@ -434,7 +446,7 @@ def test_build_spi_support_channel_fails_on_undeclared_zero_weight_row() -> None person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=3, + expected_spi_households=3, zero_weight_declarations=( UKZeroWeightStratumDeclaration( name="nonmatching", @@ -447,23 +459,31 @@ def test_build_spi_support_channel_fails_on_undeclared_zero_weight_row() -> None def test_build_spi_support_channel_fails_missing_positive_region_support() -> None: - with pytest.raises(ValueError, match="positive-mass UK base stratum"): + # Each household is its own region here, and seed 0 draws household 3 out + # of the share at 2/3, so SCOTLAND keeps base mass and no copy. The + # refusal counts strata and names none (microcosm#931). + with pytest.raises(ValueError, match="positive-mass UK base stratum") as caught: build_uk_spi_support_channel( person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=2, - seed=1, + expected_spi_households=2, + seed=0, strata_columns=("region",), zero_weight_declarations=( UKZeroWeightStratumDeclaration( - name="e7_spi_synthetic_preclone", + name="e7_spi_synthetic_share", selector={HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True}, maximum_zero_weight_rows=2, reason="synthetic test declaration", ), ), ) + assert str(caught.value) == ( + "Every positive-mass UK base stratum must have rebuilt SPI support; 1 of " + "the 3 positive-mass strata have no SPI copy." + ) + assert "SCOTLAND" not in str(caught.value) def test_replace_spi_support_builds_importance_pool_from_calibrated_base() -> None: @@ -680,7 +700,7 @@ def test_spi_source_lineage_keeps_source_support_from_doubling() -> None: person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=None, + expected_spi_households=None, source_year=2023, ) @@ -706,7 +726,7 @@ def test_spi_support_preserves_existing_rowwise_lineage_metadata() -> None: person=person_frame(), benunit=benunit_frame(), household=household, - spi_household_count=None, + expected_spi_households=None, source_year=2023, ) @@ -730,12 +750,12 @@ def test_spi_support_preserves_existing_rowwise_lineage_metadata() -> None: assert result.household["clone_index"].tolist() == [0, 1, 2, 0, 1, 2] -def test_spi_support_refuses_to_run_twice_or_oversample() -> None: +def test_spi_support_refuses_to_run_twice_and_caps_the_share_at_one() -> None: result = create_uk_spi_support_tables( person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=1, + expected_spi_households=1, ) with pytest.raises(ValueError, match="should run exactly once"): @@ -743,16 +763,27 @@ def test_spi_support_refuses_to_run_twice_or_oversample() -> None: person=result.person, benunit=result.benunit, household=result.household, - spi_household_count=1, + expected_spi_households=1, ) - with pytest.raises(ValueError, match="cannot exceed"): - create_uk_spi_support_tables( - person=person_frame(), - benunit=benunit_frame(), - household=household_frame(), - spi_household_count=4, - ) + # An expected count above the source households is a share of one: every + # household is copied, where the positional draw refused (microcosm#931). + capped = create_uk_spi_support_tables( + person=person_frame(), + benunit=benunit_frame(), + household=household_frame(), + expected_spi_households=4, + ) + assert capped.spi_inclusion_probability == 1.0 + assert capped.spi_household_ids == (1, 2, 3) + for value in (0, -1.0, float("nan"), float("inf"), True, "3"): + with pytest.raises(ValueError, match="expected_spi_households must be"): + create_uk_spi_support_tables( + person=person_frame(), + benunit=benunit_frame(), + household=household_frame(), + expected_spi_households=value, + ) def test_spi_fill_rejects_missing_source_ids() -> None: @@ -760,7 +791,7 @@ def test_spi_fill_rejects_missing_source_ids() -> None: person=person_frame(), benunit=benunit_frame(), household=household_frame(), - spi_household_count=None, + expected_spi_households=None, ) donor = pd.DataFrame( { @@ -805,7 +836,7 @@ def test_spi_support_input_refusals_count_rows_and_name_no_id() -> None: person=person, benunit=benunit_frame(), household=household_frame(), - spi_household_count=None, + expected_spi_households=None, ) assert "on 1 person row(s)" in str(caught.value) assert "777777" not in str(caught.value) @@ -817,7 +848,7 @@ def test_spi_support_input_refusals_count_rows_and_name_no_id() -> None: person=person_frame(), benunit=benunit_frame(), household=household, - spi_household_count=None, + expected_spi_households=None, ) assert str(caught.value) == ( "household.household_id must be unique; 1 row(s) repeat an earlier row's value." @@ -864,7 +895,7 @@ def test_build_spi_support_channel_gives_pension_age_households_their_own_share( person=person, benunit=benunit, household=household, - spi_household_count=4, + expected_spi_households=4, seed=42, source_year=2023, strata_columns=("region",), @@ -907,7 +938,7 @@ def test_build_spi_support_channel_without_a_pension_age_share_is_unchanged() -> person=person, benunit=benunit, household=household, - spi_household_count=4, + expected_spi_households=4, seed=42, source_year=2023, strata_columns=("region",), @@ -943,8 +974,509 @@ def test_build_spi_support_channel_refuses_invalid_pension_age_arguments( person=person, benunit=benunit, household=household, - spi_household_count=4, + expected_spi_households=4, seed=42, source_year=2023, **arguments, ) + + +# The keyed Bernoulli share (microcosm#931). Every frame below is synthetic. + +_SHARE_RULE = {"rule": spi_support.SPI_SHARE_GATE_RULE, "sigma": 4} +_SHARE_GATE = { + "name": "e7_spi_synthetic_share", + "selector": {HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: True}, + "maximum_zero_weight_rows_rule": _SHARE_RULE, + "reason": "synthetic share gate", +} +_REGIONS = np.asarray(["LONDON", "WALES", "SCOTLAND", "NORTH_EAST"]) + + +def _share_frames(n_households: int, *, seed: int = 0): + """``n_households`` one-person households over four regions.""" + + rng = np.random.default_rng(seed) + ids = np.arange(1, n_households + 1, dtype="int64") + household = pd.DataFrame( + { + "household_id": ids, + "household_weight": rng.uniform(50.0, 500.0, n_households), + "region": _REGIONS[ids % 4], + } + ) + person = pd.DataFrame( + { + "person_id": ids * 1_000 + 1, + "person_household_id": ids, + "person_benunit_id": ids * 100 + 1, + "age": 30 + ids % 50, + } + ) + benunit = pd.DataFrame({"benunit_id": ids * 100 + 1}) + return person, benunit, household + + +def _cloned_share_tables(n_households: int, *, n_clones: int): + """The share frames as ``n_clones`` geographic clones, clone-major.""" + + from microcosm.build.uk_runtime.national_frame import uk_national_frame + from microcosm.frame import engine_tables + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + person, benunit, household = _share_frames(n_households) + frame = uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2024", + weight_kind=WeightKind.DESIGN, + ) + tables = engine_tables(clone_first_frame(frame, n_clones)) + return tables["person"], tables["benunit"], tables["household"] + + +def test_the_share_draw_reads_each_households_own_id_alone() -> None: + from microcosm.build.stochastic_assignment import stable_identity_uniforms + + ids = np.arange(1, 2_001, dtype="int64") + drawn = spi_support.spi_share_inclusion(ids, inclusion_probability=0.3, seed=42) + assert 0 < int(drawn.sum()) < ids.size + # Each household is drawn when its keyed uniform falls below the share. + np.testing.assert_array_equal( + drawn, + stable_identity_uniforms( + ids, seed=42, salt=spi_support.SPI_SUPPORT_INCLUSION_SALT + ) + < 0.3, + ) + # Permuting the rows permutes the draws with them. + order = np.random.default_rng(7).permutation(ids.size) + np.testing.assert_array_equal( + spi_support.spi_share_inclusion(ids[order], inclusion_probability=0.3, seed=42), + drawn[order], + ) + # Unrelated households appended at the same probability leave every + # existing household's draw where it was (the stage's probability divides + # by the source count, so it is held fixed here). + more = np.concatenate([ids, np.arange(5_001, 6_001, dtype="int64")]) + np.testing.assert_array_equal( + spi_support.spi_share_inclusion(more, inclusion_probability=0.3, seed=42)[ + : ids.size + ], + drawn, + ) + assert not np.array_equal( + spi_support.spi_share_inclusion(ids, inclusion_probability=0.3, seed=43), drawn + ) + with pytest.raises(TypeError, match="integer ids"): + spi_support.spi_share_inclusion( + ids.astype(float), inclusion_probability=0.3, seed=42 + ) + with pytest.raises(ValueError, match="each record once"): + spi_support.spi_share_inclusion( + np.r_[ids, ids[:1]], inclusion_probability=0.3, seed=42 + ) + for probability in (0.0, -0.1, 1.5, float("nan")): + with pytest.raises(ValueError, match="must lie in"): + spi_support.spi_share_inclusion( + ids, inclusion_probability=probability, seed=42 + ) + + +def test_a_row_permutation_leaves_the_stage_selection_unchanged() -> None: + person, benunit, household = _share_frames(600) + first = create_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household, + expected_spi_households=150, + ) + order = np.random.default_rng(3).permutation(len(household)) + shuffled_household = household.iloc[order].reset_index(drop=True) + shuffled = create_uk_spi_support_tables( + person=person.iloc[::-1].reset_index(drop=True), + benunit=benunit, + household=shuffled_household, + expected_spi_households=150, + ) + + assert first.spi_inclusion_probability == shuffled.spi_inclusion_probability + assert first.spi_inclusion_probability == 0.25 + assert set(shuffled.spi_household_ids) == set(first.spi_household_ids) + # The copies keep the frame's order. + position = { + value: index for index, value in enumerate(shuffled_household["household_id"]) + } + assert list(shuffled.spi_household_ids) == sorted( + shuffled.spi_household_ids, key=position.__getitem__ + ) + + +def test_the_share_count_stays_inside_the_four_sigma_band() -> None: + n, share = 1_000, 0.3 + ids = np.arange(1, n + 1, dtype="int64") + band = n * binomial_share_tolerance(target=share, rows=n, sigma=4.0) + counts = [ + int( + spi_support.spi_share_inclusion( + ids, inclusion_probability=share, seed=seed + ).sum() + ) + for seed in range(200) + ] + + assert all(abs(count - n * share) <= band for count in counts) + # A draw, not a quota: the count varies, around n x share. + assert len(set(counts)) > 10 + assert abs(np.mean(counts) - n * share) <= 4.0 * math.sqrt( + n * share * (1.0 - share) / len(counts) + ) + + +def test_a_share_of_one_copies_every_household_whatever_its_uniform( + monkeypatch, +) -> None: + ids = np.arange(1, 11, dtype="int64") + # A keyed uniform is an integer over 2**64, which can round up to exactly + # one; at a share of one the household is copied all the same. + monkeypatch.setattr( + spi_support, + "stable_identity_uniforms", + lambda keys, *, seed, salt: np.ones(len(keys)), + ) + assert spi_support.spi_share_inclusion(ids, inclusion_probability=1.0, seed=0).all() + assert not spi_support.spi_share_inclusion( + ids, inclusion_probability=0.999, seed=0 + ).any() + person, benunit, household = _share_frames(10) + result = create_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household, + expected_spi_households=10, + ) + assert result.spi_inclusion_probability == 1.0 + assert result.spi_household_ids == tuple(range(1, 11)) + + +def test_the_share_gate_resolves_to_the_share_plus_its_binomial_band() -> None: + n, share = 1_000, 0.3 + maximum = spi_support.spi_share_gate_maximum( + _SHARE_RULE, inclusion_probability=share, household_rows=n + ) + + assert maximum == math.ceil(n * (share + 4.0 * math.sqrt(share * 0.7 / n))) + assert ( + spi_support.spi_share_gate_maximum( + _SHARE_RULE, inclusion_probability=1.0, household_rows=n + ) + == n + ) + # The unchanged gate passes the resolved maximum and refuses one more. + for zero_rows, passed in ((maximum, True), (maximum + 1, False)): + synthetic = np.arange(n) < zero_rows + gate = uk_zero_weight_strata_gate( + pd.DataFrame( + { + "household_id": np.arange(1, n + 1), + "household_weight": np.where(synthetic, 0.0, 1.0), + HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN: synthetic, + } + ), + declarations=spi_support._coerce_zero_weight_declarations( + (_SHARE_GATE,), inclusion_probability=share, household_rows=n + ), + ) + assert gate.passed is passed + + +@pytest.mark.parametrize( + ("declaration", "probability", "message"), + [ + ( + {**_SHARE_GATE, "maximum_zero_weight_rows_rule": {"rule": "normal"}}, + 0.3, + "maximum rule must be", + ), + ( + { + **_SHARE_GATE, + "maximum_zero_weight_rows_rule": {**_SHARE_RULE, "sigma": 0}, + }, + 0.3, + "sigma must be positive", + ), + ( + { + **_SHARE_GATE, + "maximum_zero_weight_rows_rule": {**_SHARE_RULE, "sigma": True}, + }, + 0.3, + "sigma must be positive", + ), + ( + {**_SHARE_GATE, "maximum_zero_weight_rows": 10}, + 0.3, + "both a maximum and a maximum rule", + ), + (_SHARE_GATE, None, "named copies are not a share"), + ], +) +def test_the_share_gate_refuses_a_malformed_or_unresolvable_rule( + declaration, probability, message: str +) -> None: + with pytest.raises(ValueError, match=message): + spi_support._coerce_zero_weight_declarations( + (declaration,), inclusion_probability=probability, household_rows=100 + ) + + +def test_the_share_gate_passes_each_draw_and_refuses_one_copy_above_it( + monkeypatch, +) -> None: + person, benunit, household = _share_frames(400) + band = 400 * binomial_share_tolerance(target=0.25, rows=400, sigma=4.0) + for seed in range(40): + result = build_uk_spi_support_channel( + person, + benunit, + household, + expected_spi_households=100, + seed=seed, + zero_weight_declarations=(_SHARE_GATE,), + ) + assert abs(result.n_spi_households - 100) <= band + maximum = spi_support.spi_share_gate_maximum( + _SHARE_RULE, inclusion_probability=0.25, household_rows=400 + ) + # One copy more than the bound, injected through the draw, is refused, by + # count. + monkeypatch.setattr( + spi_support, + "spi_share_inclusion", + lambda ids, *, inclusion_probability, seed: np.arange(len(ids)) <= maximum, + ) + with pytest.raises(ValueError) as caught: + build_uk_spi_support_channel( + person, + benunit, + household, + expected_spi_households=100, + zero_weight_declarations=(_SHARE_GATE,), + ) + assert str(caught.value) == ( + "UK SPI support zero-weight gate failed: ['e7_spi_synthetic_share: " + f"{maximum + 1} zero-weight rows exceed the declared maximum {maximum}.']." + ) + + +def test_an_unmatched_zero_weight_base_row_still_fails_the_share_gate() -> None: + person, benunit, household = _share_frames(400) + household.loc[0, "household_weight"] = 0.0 + + with pytest.raises(ValueError, match=r"1 zero-weight household row\(s\) match no"): + build_uk_spi_support_channel( + person, + benunit, + household, + expected_spi_households=100, + zero_weight_declarations=(_SHARE_GATE,), + ) + + +def test_the_share_counts_distinct_sources_and_refuses_an_unsourced_row() -> None: + person, benunit, household = _share_frames(8) + # Two rows per source: the share divides by the four sources. + household["source_household_id"] = (household["household_id"] + 1) // 2 + result = create_uk_spi_support_tables( + person=person, benunit=benunit, household=household, expected_spi_households=2 + ) + assert result.spi_source_households == 4 + assert result.spi_inclusion_probability == 0.5 + + household["source_household_id"] = household["source_household_id"].astype("Int64") + household.loc[3, "source_household_id"] = pd.NA + with pytest.raises(ValueError) as caught: + create_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household, + expected_spi_households=2, + ) + assert str(caught.value) == ( + "1 household row(s) carry no source_household_id; the SPI share counts " + "distinct source households." + ) + + +def test_the_clone_stratum_reads_the_lineages_geographic_index() -> None: + from microcosm.build.uk_runtime.cgt_support import _GEOGRAPHIC_CLONE_INDEX_COLUMN + from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_SUPPORT_BRANCH, + UK_COPY_LINEAGE, + ) + from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column + + (branch,) = [ + branch + for branch in UK_COPY_LINEAGE.branches + if branch.name == GEOGRAPHIC_SUPPORT_BRANCH + ] + assert ( + spi_support.SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN + == _GEOGRAPHIC_CLONE_INDEX_COLUMN + == ladder_clone_index_column("household") + == branch.index + ) + + +def test_one_indexed_clone_allocates_bit_for_bit_as_no_index() -> None: + person, benunit, household = _share_frames(400) + arguments = { + "expected_spi_households": 100, + "pension_age_share": 0.2, + "state_pension_age": 66, + } + plain = build_uk_spi_support_channel(person, benunit, household, **arguments) + indexed = build_uk_spi_support_channel( + person, + benunit, + household.assign(household_clone_index=np.zeros(len(household), "int64")), + **arguments, + ) + + assert indexed.spi_household_ids == plain.spi_household_ids + np.testing.assert_array_equal( + indexed.household["household_weight"].to_numpy(), + plain.household["household_weight"].to_numpy(), + ) + assert indexed.mass_log == plain.mass_log + assert spi_support.SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN not in indexed.household + + +def test_each_geographic_clone_keeps_its_own_spi_prior_mass() -> None: + person, benunit, household = _cloned_share_tables(400, n_clones=3) + result = build_uk_spi_support_channel( + person, benunit, household, expected_spi_households=100 + ) + out = result.household + spi = out[support_channel_column("household")].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL) + + # Every clone and region keeps its mass, half of it on its own copies. + for (clone, region), group in household.groupby( + ["household_clone_index", "region"] + ): + members = out["household_clone_index"].eq(clone) & out["region"].eq(region) + mass = float(group["household_weight"].sum()) + assert float(out.loc[members, "household_weight"].sum()) == pytest.approx( + mass, rel=1e-12 + ) + assert float(out.loc[members & spi, "household_weight"].sum()) == ( + pytest.approx(0.5 * mass, rel=1e-12) + ) + assert spi_support.SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN not in out + + # Without the per-clone stratum the region's copies share one weight, so + # a clone's copies carry the region's share of its copies, not of its mass. + pooled = build_uk_spi_support_channel( + person, + benunit, + household.drop(columns=["household_clone_index"]), + expected_spi_households=100, + ).household + pooled_spi = pooled[support_channel_column("household")].eq( + SPI_SYNTHETIC_SUPPORT_CHANNEL + ) + clone_of = household.set_index("household_id")["household_clone_index"] + pooled_clone = pooled["household_source_id"].map(clone_of) + pooled_shares = [ + float(pooled.loc[pooled_spi & pooled_clone.eq(clone), "household_weight"].sum()) + / float(household.loc[clone_of.to_numpy() == clone, "household_weight"].sum()) + for clone in range(3) + ] + assert max(abs(share - 0.5) for share in pooled_shares) > 1e-3 + + +def test_an_empty_clone_stratum_is_refused_by_count() -> None: + person, benunit, household = _cloned_share_tables(8, n_clones=2) + # Every clone-0 household is copied, and clone 1's but its LONDON ones: + # LONDON keeps copies in clone 0 only. + london = household["region"].eq("LONDON").to_numpy() + clone = household["household_clone_index"].to_numpy() + named = tuple(household.loc[~(london & (clone == 1)), "household_id"]) + stacked = create_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household, + selected_household_ids=named, + ) + arguments = { + "spi_prior_mass_share": 0.5, + "input_weight_kind": WeightKind.DESIGN, + "mass_log": (), + "strata_columns": ("region",), + } + + with pytest.raises(ValueError) as caught: + spi_support._allocate_spi_prior_mass(stacked, **arguments) + assert str(caught.value) == ( + "Every positive-mass UK base stratum must have rebuilt SPI support; 1 of " + "the 8 positive-mass strata have no SPI copy." + ) + # Pooled over clones, the region has copies. + pooled = create_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household.drop(columns=["household_clone_index"]), + selected_household_ids=named, + ) + spi_support._allocate_spi_prior_mass(pooled, **arguments) + + +@pytest.mark.parametrize( + ("values", "message"), + [ + ( + np.asarray([0.0, 1.0] * 4), + "The SPI prior allocation stratifies geographic clones on an integer " + "'household_clone_index' column; it has dtype float64.", + ), + ( + pd.array([0, None, 1, 0, 1, 0, 1, 0], dtype="Int64"), + "1 household row(s) carry no 'household_clone_index'; the SPI prior " + "allocation stratifies each geographic clone.", + ), + ( + np.asarray([0, -1, 1, -2, 0, 1, 0, 1], dtype="int64"), + "2 household row(s) carry a negative 'household_clone_index'; the SPI " + "prior allocation stratifies each geographic clone.", + ), + ], +) +def test_the_clone_stratum_refuses_a_malformed_index_by_count( + values, message: str +) -> None: + person, benunit, household = _share_frames(8) + with pytest.raises(ValueError) as caught: + build_uk_spi_support_channel( + person, + benunit, + household.assign(household_clone_index=values), + expected_spi_households=8, + ) + assert str(caught.value) == message + + +def test_the_derived_clone_stratum_is_not_a_declared_stratum() -> None: + person, benunit, household = _share_frames(8) + column = spi_support.SPI_GEOGRAPHIC_CLONE_STRATUM_COLUMN + + with pytest.raises(ValueError, match="derived by the allocation"): + build_uk_spi_support_channel( + person, + benunit, + household.assign(**{column: 0}), + expected_spi_households=8, + strata_columns=("region", column), + ) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 9b54b985d2..1fc680e4fe 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -7475,7 +7475,7 @@ "source_household_id", "source_year" ], - "notes": "Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates prior mass with exact national conservation within each region, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069).", + "notes": "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's draw depends on its own id alone. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069).", "operations": [ { "channels": { @@ -7483,18 +7483,22 @@ "synthetic": "spi" }, "count": 10000, - "draw": "uniform_without_replacement", + "draw": "keyed_bernoulli_share: stable_identity_uniforms(household_id, seed, salt) < min(1, count x sample fraction / distinct source households)", "flag_column": "household_is_spi_synthetic", "id_remapping": "copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11)", "kind": "stack_zero_weight_donors", + "salt": "spi_support_channel:inclusion", "seed": 42 }, { "declarations": [ { - "maximum_zero_weight_rows": 10000, - "name": "e7_spi_synthetic_preclone", - "reason": "E7 stacks exactly 10,000 zero-weight SPI-synthetic households on the raw pre-clone UK FRS spine.", + "maximum_zero_weight_rows_rule": { + "rule": "binomial_share_band", + "sigma": 4 + }, + "name": "e7_spi_synthetic_share", + "reason": "E7 stacks a keyed Bernoulli share of the raw UK FRS spine's households as zero-weight SPI-synthetic households, so their count is binomial; it may exceed the share of the household rows entering the stage by at most the repo's four-sigma binomial band, floored at one row.", "selector": { "household_is_spi_synthetic": true } @@ -7505,6 +7509,7 @@ }, { "conservation": "exact_total", + "geographic_clone_stratum": "household_clone_index", "kind": "allocate_zero_weight_prior_mass", "pension_age_share": 0.2, "pension_age_stratum": "any_member_at_or_over_state_pension_age", @@ -7531,7 +7536,7 @@ "source_household_key" ], "rewrites": [], - "source": "Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning.", + "source": "Synthetic SPI support channel drawn from the raw FRS spine as a keyed Bernoulli share, each household on its own identity-keyed uniform.", "stage": "spi_support_channel", "survey": "Family Resources Survey 2024-25" }, diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index fbf4c2979f..80b081846c 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"48b7648953aabc721c6672038770b18a1ff7c308d71bd982770b8a3fe259280f","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"85696cd009ce349a132f753c55cb2168ed2f548de9f628ce74a71e99a0e9e6ba","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"fec5bddc81b70da31dcf9a301405049b71b11113bcdd2c2c5950efc8e3a4dc18","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"85696cd009ce349a132f753c55cb2168ed2f548de9f628ce74a71e99a0e9e6ba","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"fec5bddc81b70da31dcf9a301405049b71b11113bcdd2c2c5950efc8e3a4dc18","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} From 8137a1b053e92b7a30ac1e0900356da1551ca172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:07:56 +0000 Subject: [PATCH 027/108] Seat the SPI income band donors within each geographic clone (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 17 seated its donors in one walk over every candidate, and sized the seats on the eligible households counted in rows. Once (b) clones households at the spine's start, the walk would pool the clones, and the seats and the 15% household-share cap would grow with K, so clone 0 of a K=15 spine would not seat the K=1 donors. As María ruled on 7 Nov, the stage now runs today's walk inside each geographic clone, with one common mass scale. - Seating: where household_clone_index is present, the candidates are grouped by their household's clone; without it there is one group. The eligible households are counted in distinct source_household_id values, so the seats and the cap (spi_income_band_donor_seats) are the same at every K. Today's walk runs once per clone with those seats and the same person-keyed uniforms, Efraimidis-Spirakis keys and lexsort, so clone 0, whose ids are the one-clone ids, seats exactly the one-clone donors. The per-band eligibility check runs per clone and refuses by count. selected_household and carrier_person stay shared, since ids differ across clones. A non-integer, missing or negative clone index, or a gap in 0 to K - 1, is refused by count. - Weights and funding: each donor weighs (T_b / count_b) / K. Its funding stratum is its region within its clone: _funding_strata appends the clone where the column is present, and the manifest's funding_strata stays ['region']. So every clone's regions conserve their own mass. One mass scale, the minimum over every clone-region stratum, serves every clone (ruling 2), so each band keeps one donor weight, and stage_health's band-mass reconciliation and the identity tool's E8 replay hold unchanged. The exact-total correction still moves one incumbent row and never a donor, so a band's donors stay equal bit for bit at any K. With exact replicas it lands on clone 0, whose weights then nest to rtol 1e-12, not bitwise. The MassChangeRecord reason is unchanged, so uk_frame_content_identity and the coverage manifest's required_mass_change_reason do not move. - Receipt: new key clone_count, and funding_strata lists household_clone_index where the column is present (the stratum labels read region|clone). expected_donor_households, expected_donor_count and the donor counts cover every clone; donor_weight and planned_donor_weight are one clone's. At the receipts' floor of three, as the design review asked, both eligible counts go through rounded_count over distinct true sources in place of the exact ones; seating_scale reads 'full' or 'capped', since an exact scale below one gives back the eligible count; a band's carrier_mean_age reads '<3' below three carrier source households; and carrier_region_counts and the funding rows' donor_households are rounded. Besides the tests, stage_health reads seating_scale and the identity tool copies it; nothing else reads the changed fields. - Gate: the band-donor check refuses a receipt without a positive integer clone_count. It requires donor weight x K within the maximum, donors that split evenly across the clones, and at full seating donors equal to the receipt's plan (K x one clone's) and at least K x 120. The band-mass check is unchanged. At K = 1 the checks and messages are as they were, except that the seating scale is read as 'full' or 'capped' rather than a number in (0, 1]. - Texts: the draw, weight_rule, funding, seating_scale and mass_scale constants and the stage dictionary, sources.yaml (and the stage's notes) and source_stages.json. The design's draw wording named "geographic_expand", which nothing in the code defines, so the draw names the node that writes the clone index, uk.full.expand. - Bindings: the derived tests in test_uk_graph.py pass with no change. The stage already counts by lineage (it is in _COUNTING_STAGES) and binds spi_support and stochastic_assignment, and the clone column's name is a string constant. What moves: no value at K = 1. On 48 synthetic stage-16-then-17 runs (8 frames, 3 band configurations, 2 seeds; two of the frames carry a one-clone index), the seating, carriers, every household weight (hex), donor weights, mass and seating scales, funding factors, mass log and content identity after the stage equal the parent commit's. The receipt gains clone_count and takes the floor-3 forms above. spi_income_band_donors and every node after it re-key, the spine gates included, as does any kernel keyed on the spec or the coverage manifest. Regenerated: source_stages.json, the coverage manifest's 25 source digests, the stage's declaration in the H2 fixture (fixture.json) and its contract hash (uk_spine.json). Tests: - K = 1: the existing seating, carrier, weight and receipt expectations hold, with seating_scale now 'full' or 'capped'. A pinned test checks uk_frame_content_identity after stage 17 on three synthetic frames (full seating, capped seating, scaled mass) against the parent commit's values. A one-clone index seats, weighs and funds bit for bit as no index. - K = 3 and 7, with clone-first frames run through stage 16: clone 0 seats the K = 1 households, bands and carriers; every clone seats the same seats, and the other clones seat their own. K x clone-0 weights, donors included, equal K = 1's at rtol 1e-12, and so do clone 0's funding factors. - A capped seating scale is the same at K = 1 and K = 3. Each (clone, region) conserves its mass. Where one clone's region binds, its factor is 0.5, the one scale serves every clone and band, and the gate passes the scaled receipt. - Refusals: a clone without candidates, and a float, missing, negative or gapped clone index, each by count. A row permutation at K = 3 leaves the seating unchanged. - Receipt: the eligible counts are rounded, checked against an independent recount; the funding and region counts are rounded; the key sets are pinned. The mean age reads '<3' at K = 1 and where three clones' carriers share one source household. - Gate: the K = 3 receipt passes, a dropped clone_count raises, and a misread one fails. Synthetic K = 3 evidence passes and fails on weight x K, uneven splits and the scaled minimum, with the K = 1 messages pinned. - The identity tool's E8 replays a K = 3 seating with no problems and flags a reweighted clone-2 donor. - 17 mutations on scratch copies of the source each fail a test. The touched engine-free files pass, with ruff. source_stages.json and the coverage manifest are current. The H2 fixture regenerates and passes its check, and H2 parity passes in the engine slot. Co-Authored-By: Claude Opus 5.5 --- .../931-band-donors-per-clone.changed.md | 7 + .../uk/release_input_coverage_manifest.json | 50 +- .../src/microcosm/build/uk/source_stages.json | 12 +- .../src/microcosm/build/uk/spec/sources.yaml | 12 +- .../build/uk_runtime/spi_band_donors.py | 358 +++++++-- .../build/uk_runtime/stage_health.py | 78 +- .../engine_free/uk/test_uk_spi_band_donors.py | 716 +++++++++++++++++- .../engine_free/uk/test_uk_stage_health.py | 97 ++- .../parity/uk_spine/sources/fixture.json | 12 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- tools/verify_uk_identity_stability.py | 17 +- 11 files changed, 1205 insertions(+), 156 deletions(-) create mode 100644 changelog.d/931-band-donors-per-clone.changed.md diff --git a/changelog.d/931-band-donors-per-clone.changed.md b/changelog.d/931-band-donors-per-clone.changed.md new file mode 100644 index 0000000000..592ede124b --- /dev/null +++ b/changelog.d/931-band-donors-per-clone.changed.md @@ -0,0 +1,7 @@ +The SPI income band donors (`spi_income_band_donors`) are seated within each geographic clone (#931). Where the households carry `household_clone_index`, the stage runs its Efraimidis-Spirakis walk once in each clone, over that clone's candidates, with the same person-keyed uniforms, keys and tie order, and the same seats in every clone (María's rulings of 7 Nov). The seats and the household share's cap read the eligible households counted in distinct `source_household_id` values, so they do not depend on the clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, seats exactly the one-clone donors, carriers and bands. Each donor weighs its band's published taxpayers over the clone count times the band's donor count. A spine without the column, as every spine is today, is one clone. + +- **Funding.** A donor's funding stratum is its region within its clone, a derived stratum (the manifest's `funding_strata` stays `[region]`), so every clone's regions conserve their own mass. One mass scale, the smallest any clone's region allows, serves every clone, so each band keeps one donor weight and the stage gate's and the identity tool's band-mass reconciliations hold as before. The exact-total correction still lands on one incumbent row, never on a donor, so a band's donors stay equal bit for bit at any clone count; where the clones are exact replicas the corrected row is clone 0's, so clone 0's weights nest to about 1e-12, not bitwise. The mass record's reason is unchanged, so neither the frame's content identity nor the coverage manifest's restated reason moves. +- **Receipt.** The evidence gains `clone_count`, and `funding_strata` lists `household_clone_index` where the column is present (the funding rows' labels then read `region|clone`). `expected_donor_households`, `expected_donor_count` and the donor counts cover every clone; `donor_weight` and `planned_donor_weight` are one clone's weights. Under the receipts' floor of three (`receipt_counts`), the eligible household counts, in all and per band, are counted in source households and rounded in place of the exact counts; `seating_scale` reads `full` or `capped`, since an exact scale below one gives back the eligible count; a band's `carrier_mean_age` reads `<3` when its carriers come from fewer than three source households; and `carrier_region_counts` and the funding rows' `donor_households` are rounded. +- **Gate.** The `spi_income_band_donor_support` check refuses a receipt without a positive integer `clone_count`. A donor's weight times the clone count must stay within the maximum, a band's donors must split evenly across the clones, and at full seating a band seats its plan and never fewer than the clone count times the minimum. The seating scale is read as `full` or `capped`. At one clone the checks and their messages are as before, apart from the seating scale's form. +- **Refusals.** A clone short of candidates for a band is refused with the number of such clones and the fewest candidates found. A non-integer, missing or negative clone index, or an index from 0 to K - 1 that no household carries, is refused by count. +- **What moves.** At K = 1, no value: on synthetic frames the seating, the carriers, every weight, the funding factors, the mass and seating scales, the mass record and the frame's content identity after the stage are bit for bit the parent commit's, with or without a one-clone index. The receipt takes the forms above. The stage's draw, weight-rule, funding, seating-scale and mass-scale texts and its notes change in `sources.yaml`, `source_stages.json` and the stage dictionary, so the stage's contract hash, the H2 fixture's declarations and the coverage manifest's 25 source digests are regenerated. `spi_income_band_donors` and every node after it re-key, as do the spine gates and the stages that bind `stage_health`. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 13f6baeda3..34a358b664 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "193573efaa39580a2b64321306ed5d8978bcbdf4240068132d7cd4ae778d9335", + "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index fbd58cc484..947c391cbe 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -1791,11 +1791,11 @@ "minimum_donors_per_band": 120, "maximum_donor_weight": 250.0, "count_rule": "max(minimum_donors_per_band, ceil(published band taxpayers / maximum_donor_weight))", - "weight_rule": "published band taxpayers / band donor count, equal within a band", + "weight_rule": "published band taxpayers / (geographic clone count x band donor count), equal within a band", "expected_band_count": 4, "candidate_population": "raw FRS base channel claimants and partners (is_uc_claimant) aged 16 and over", "candidate_order": "person_id ascending breaks equal draw keys", - "draw": "weighted_without_replacement_identity_keyed: Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", + "draw": "weighted_without_replacement_identity_keyed within each geographic clone (one before uk.full.expand): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", "propensity": "SPI 2022-23 FACT-weighted band share by region, sex and age band", "tape_preparation": "as hmrc_spi_income_spine prepares the tape: its stage-1 seed and draw_spi_donor_ages_by_population", "tape_preparation_seed": 42, @@ -1806,14 +1806,14 @@ "flag_column": "household_is_spi_income_band_donor", "band_column": "spi_income_band_donor_lower_bound", "carrier_column": "person_is_spi_income_band_carrier", - "funding": "stratum reallocation: every incumbent household of the stratum a donor's source household belongs to is scaled by 1 - donor mass / stratum mass", + "funding": "stratum reallocation per geographic clone: every incumbent household of the stratum a donor's source household belongs to, in the donor's clone, is scaled by 1 - donor mass / stratum mass", "funding_strata": [ "region" ], "maximum_reallocation_share": 0.5, "maximum_donor_household_share": 0.15, - "seating_scale": "min(1, maximum_donor_household_share x eligible base households / expected donor count); band counts floored, at least one per band", - "mass_scale": "min(1, min over funding strata of maximum_reallocation_share x stratum mass / seated donor mass); every band's weight scaled alike", + "seating_scale": "min(1, maximum_donor_household_share x eligible base source households / expected donor count of one geographic clone); band counts floored, at least one per band", + "mass_scale": "min(1, min over the funding strata of every geographic clone of maximum_reallocation_share x stratum mass / seated donor mass); one scale for every band and clone", "never_zero_weight": true, "weight_kind_out": "importance", "conservation": "exact_total", @@ -1841,7 +1841,7 @@ "nonnegative_outputs": [ "spi_income_band_donor_lower_bound" ], - "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves)." + "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, with the seats counted in distinct source households, each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves)." }, { "stage": "hmrc_spi_income_spine", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 391c83c596..c17a0b8938 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -1606,11 +1606,11 @@ stages: minimum_donors_per_band: 120 maximum_donor_weight: 250.0 count_rule: max(minimum_donors_per_band, ceil(published band taxpayers / maximum_donor_weight)) - weight_rule: published band taxpayers / band donor count, equal within a band + weight_rule: published band taxpayers / (geographic clone count x band donor count), equal within a band expected_band_count: 4 candidate_population: raw FRS base channel claimants and partners (is_uc_claimant) aged 16 and over candidate_order: person_id ascending breaks equal draw keys - draw: 'weighted_without_replacement_identity_keyed: Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first' + draw: 'weighted_without_replacement_identity_keyed within each geographic clone (one before uk.full.expand): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first' propensity: SPI 2022-23 FACT-weighted band share by region, sex and age band tape_preparation: 'as hmrc_spi_income_spine prepares the tape: its stage-1 seed and draw_spi_donor_ages_by_population' tape_preparation_seed: 42 @@ -1621,13 +1621,13 @@ stages: flag_column: household_is_spi_income_band_donor band_column: spi_income_band_donor_lower_bound carrier_column: person_is_spi_income_band_carrier - funding: 'stratum reallocation: every incumbent household of the stratum a donor''s source household belongs to is scaled by 1 - donor mass / stratum mass' + funding: 'stratum reallocation per geographic clone: every incumbent household of the stratum a donor''s source household belongs to, in the donor''s clone, is scaled by 1 - donor mass / stratum mass' funding_strata: - region maximum_reallocation_share: 0.5 maximum_donor_household_share: 0.15 - seating_scale: min(1, maximum_donor_household_share x eligible base households / expected donor count); band counts floored, at least one per band - mass_scale: min(1, min over funding strata of maximum_reallocation_share x stratum mass / seated donor mass); every band's weight scaled alike + seating_scale: min(1, maximum_donor_household_share x eligible base source households / expected donor count of one geographic clone); band counts floored, at least one per band + mass_scale: min(1, min over the funding strata of every geographic clone of maximum_reallocation_share x stratum mass / seated donor mass); one scale for every band and clone never_zero_weight: true weight_kind_out: importance conservation: exact_total @@ -1650,7 +1650,7 @@ stages: - household_source_id nonnegative_outputs: - spi_income_band_donor_lower_bound - notes: Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves). + notes: Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, with the seats counted in distinct source households, each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves). - stage: hmrc_spi_income_spine survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py index 36cb38b6ad..807208fc50 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py @@ -22,6 +22,15 @@ from the households of their region. Total household mass and every funding stratum's mass are conserved; the seats are drawn from identity-keyed uniforms, so the draw can be reconstructed from the ids. + +On a clone-first spine every household carries its geographic clone +(``household_clone_index``, microcosm#931). Each clone then seats its own +donors by the same walk over its own candidates, with the same seats per band, +counted in distinct source households, so clone 0 seats exactly what a +one-clone spine seats. Every donor weighs its band's weight over the clone +count, and each clone's regions fund their own donors at one mass scale common +to every clone. A spine without the column is one clone, as every spine is +today: ``uk.full.expand`` writes the index after the spine. """ from __future__ import annotations @@ -42,6 +51,13 @@ uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.receipt_counts import ( + UK_DISCLOSURE_FLOOR, + UK_WITHHELD, + distinct_count, + rounded_count, + source_household_ids, +) from microcosm.build.uk_runtime.spi_income import ( SPI_MINIMUM_RECIPIENT_AGE, SPI_RECIPIENT_ROLE_COLUMN, @@ -57,6 +73,7 @@ _PERSON_ID_COLUMNS, BASE_FRS_SUPPORT_CHANNEL, HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, + SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN, SPI_SYNTHETIC_SUPPORT_CHANNEL, _clone_support_frame, _exact_total_correction, @@ -74,20 +91,24 @@ 1_000_000, 2_000_000, ) -#: Every band seats at least this many donor households, and enough of them -#: that no donor starts above the maximum weight: the count is -#: ``max(minimum, ceil(published taxpayers / maximum weight))`` and the weight -#: the published taxpayers over the count, equal within the band. +#: Every band seats at least this many donor households in each geographic +#: clone, and enough of them that no donor starts above the maximum weight: +#: the count is ``max(minimum, ceil(published taxpayers / maximum weight))``, +#: and every donor of the band weighs the published taxpayers over the clone +#: count times the count. SPI_INCOME_BAND_MINIMUM_DONORS = 120 SPI_INCOME_BAND_MAXIMUM_DONOR_WEIGHT = 250.0 #: Small frames: donors may take at most this share of the eligible base -#: households (the seating scale), and of any funding stratum's mass (the mass -#: scale). Both scales are one on the full frame, where the published band -#: masses are met exactly. +#: households, counted in distinct source households so the share does not +#: depend on the clone count (the seating scale), and of any funding stratum's +#: mass (the mass scale). Both scales are one on the full frame, where the +#: published band masses are met exactly. SPI_INCOME_BAND_MAXIMUM_DONOR_HOUSEHOLD_SHARE = 0.15 SPI_INCOME_BAND_MAXIMUM_REALLOCATION_SHARE = 0.5 #: The stratum a donor's mass is taken from: the households of its source -#: household's region, base and synthetic channel alike. +#: household's region, base and synthetic channel alike. Where the frame +#: carries geographic clones the stratum is the region within the donor's +#: clone, a derived stratum this declaration does not list (microcosm#931). SPI_INCOME_BAND_FUNDING_STRATA: tuple[str, ...] = ("region",) SPI_INCOME_BAND_DONOR_SEED = 3 #: The tape is prepared exactly as ``hmrc_spi_income_spine`` prepares it (its @@ -129,26 +150,35 @@ "maximum_donor_weight))" ) SPI_INCOME_BAND_DONOR_WEIGHT_RULE = ( - "published band taxpayers / band donor count, equal within a band" + "published band taxpayers / (geographic clone count x band donor count), " + "equal within a band" ) SPI_INCOME_BAND_DONOR_DRAW = ( - "weighted_without_replacement_identity_keyed: Efraimidis-Spirakis keys " - "log(u) / (household weight x propensity) on stable_identity_uniforms(" - "person_id, seed, salt:band); the largest keys are seated, one carrier per " - "household, the highest band first" + "weighted_without_replacement_identity_keyed within each geographic clone " + "(one before uk.full.expand): Efraimidis-Spirakis keys log(u) / " + "(household weight x propensity) on stable_identity_uniforms(person_id, " + "seed, salt:band); the largest keys are seated, one carrier per household, " + "the highest band first" ) SPI_INCOME_BAND_DONOR_FUNDING = ( - "stratum reallocation: every incumbent household of the stratum a donor's " - "source household belongs to is scaled by 1 - donor mass / stratum mass" + "stratum reallocation per geographic clone: every incumbent household of " + "the stratum a donor's source household belongs to, in the donor's clone, " + "is scaled by 1 - donor mass / stratum mass" ) SPI_INCOME_BAND_DONOR_SEATING_SCALE = ( - "min(1, maximum_donor_household_share x eligible base households / expected " - "donor count); band counts floored, at least one per band" + "min(1, maximum_donor_household_share x eligible base source households / " + "expected donor count of one geographic clone); band counts floored, at " + "least one per band" ) SPI_INCOME_BAND_DONOR_MASS_SCALE = ( - "min(1, min over funding strata of maximum_reallocation_share x stratum " - "mass / seated donor mass); every band's weight scaled alike" + "min(1, min over the funding strata of every geographic clone of " + "maximum_reallocation_share x stratum mass / seated donor mass); one scale " + "for every band and clone" ) +#: The mass-change record's reason. It is hashed into the frame's content +#: identity and restated by the release input coverage manifest, so it stays +#: byte for byte as it was before the stage seated per clone: every clone's +#: regions conserve their mass, so every region does. SPI_INCOME_BAND_DONOR_MASS_CHANGE_REASON = ( "Reallocate the published HMRC Table 2.5 taxpayer mass of each total-income " "band from GBP 200,000 from the households of each donor's region to its " @@ -171,11 +201,22 @@ SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN, PERSON_IS_SPI_INCOME_BAND_CARRIER, ) +#: How the receipt records the seating scale: an exact scale below one gives +#: back the eligible count, so the receipt says only whether the household +#: share's cap bound (UKDS disclosure control). +SPI_INCOME_BAND_FULL_SEATING = "full" +SPI_INCOME_BAND_CAPPED_SEATING = "capped" @dataclass(frozen=True) class UKSPIIncomeBandDonorResult: - """Frame with the reserved band rows and the receipt the gate reads.""" + """Frame with the reserved band rows and the receipt the gate reads. + + ``seating_scale`` is the exact scale the seats were cut at; the receipt + records it as full or capped. ``eligible_households`` and the counts in + the band, funding and carrier rows are already in their receipt forms: + distinct source households, rounded (UKDS disclosure control). + """ frame: Frame band_rows: tuple[Mapping[str, object], ...] @@ -186,7 +227,7 @@ class UKSPIIncomeBandDonorResult: maximum_reallocation_share: float seating_scale: float mass_scale: float - eligible_households: int + eligible_households: int | str expected_donor_count: int donor_count: int reallocated_mass: float @@ -195,8 +236,10 @@ class UKSPIIncomeBandDonorResult: seed: int sample_fraction: float taxpayer_period: int - carrier_region_counts: Mapping[str, int] + carrier_region_counts: Mapping[str, int | str] propensity_cells: int + clone_count: int + funding_strata: tuple[str, ...] def evidence(self) -> dict[str, object]: return { @@ -205,8 +248,13 @@ def evidence(self) -> dict[str, object]: "maximum_donor_weight": self.maximum_donor_weight, "maximum_donor_household_share": self.maximum_donor_household_share, "maximum_reallocation_share": self.maximum_reallocation_share, - "funding_strata": list(SPI_INCOME_BAND_FUNDING_STRATA), - "seating_scale": self.seating_scale, + "clone_count": self.clone_count, + "funding_strata": list(self.funding_strata), + "seating_scale": ( + SPI_INCOME_BAND_FULL_SEATING + if self.seating_scale >= 1.0 + else SPI_INCOME_BAND_CAPPED_SEATING + ), "mass_scale": self.mass_scale, "sample_fraction": self.sample_fraction, "eligible_households": self.eligible_households, @@ -274,7 +322,9 @@ def spi_income_band_donor_seats( The scale is one while the planned donors fit inside the declared share of the eligible base households; below that every band's count is scaled alike and floored, never below one, so a small frame keeps every band - without copying most of its households. + without copying most of its households. The stage counts the eligible + households in distinct source households, so a frame's geographic clones + count once and every clone takes these seats (microcosm#931). """ expected = sum(count for count, _ in plan.values()) @@ -484,6 +534,15 @@ def stack_spi_income_band_donors( and starts at its band's equal weight. The mass the donors carry leaves the incumbent households of each donor's funding stratum in proportion, so the total and every stratum's mass are unchanged. + + Where the households carry ``household_clone_index`` (microcosm#931), the + walk runs once in each geographic clone, over that clone's candidates, + with the same seats and the same person-keyed uniforms; the seats come + from the eligible households counted in distinct source households, so + they do not depend on the clone count. Each donor weighs its band's + weight over the clone count, each clone's regions fund their own donors, + and one mass scale, the smallest any clone's region allows, serves every + clone, so each band keeps one donor weight. """ validate_uk_national_frame(frame) @@ -528,8 +587,22 @@ def stack_spi_income_band_donors( bands = sorted(plan) incoming = frame.weights_for("household") incoming_weights = np.asarray(incoming.values, dtype=float) - weight_by_household = pd.Series( - incoming_weights, index=household["household_id"].to_numpy() + household_ids = household["household_id"].to_numpy() + weight_by_household = pd.Series(incoming_weights, index=household_ids) + # The geographic clone each household belongs to (one clone without the + # index) and the survey household it was made from: the walk runs in each + # clone, and the receipt's counts and the seats count true sources. + clone_indices = _geographic_clone_indices(household) + clone_count = _geographic_clone_count(clone_indices) + clone_by_household = pd.Series( + np.zeros(len(household), dtype=np.int64) + if clone_indices is None + else clone_indices, + index=household_ids, + ) + source_by_household = pd.Series( + source_household_ids(household, stage=UK_SPI_INCOME_BAND_DONORS_STAGE_NAME), + index=household_ids, ) base_households = household.loc[ @@ -553,11 +626,22 @@ def stack_spi_income_band_donors( ) candidate_ids = candidates["person_id"].to_numpy() candidate_households = candidates["person_household_id"].to_numpy() + candidate_sources = ( + candidates["person_household_id"].map(source_by_household).to_numpy() + ) candidate_weights = ( candidates["person_household_id"].map(weight_by_household).to_numpy(float) ) if not np.isfinite(candidate_weights).all() or (candidate_weights < 0.0).any(): raise ValueError("SPI income band donor candidates need finite weights.") + # Each clone's candidate positions, in the person-id order above: with one + # clone, every candidate. + candidate_clones = ( + candidates["person_household_id"].map(clone_by_household).to_numpy(np.int64) + ) + clone_positions = [ + np.flatnonzero(candidate_clones == clone) for clone in range(clone_count) + ] # Rate of seating by weight x propensity for each band; a candidate with a # zero rate for every band can never be seated and is not eligible. @@ -570,13 +654,17 @@ def stack_spi_income_band_donors( any_rate = np.zeros(len(candidates), dtype=bool) for band in bands: any_rate |= rates[band] > 0.0 - eligible_households = int(pd.unique(candidate_households[any_rate]).size) + # Counted in distinct source households, so the seats and the household + # share's cap read the survey households, however many clones carry them. + eligible_households = distinct_count(candidate_sources, any_rate) seating_scale, seats = spi_income_band_donor_seats( plan, eligible_households=eligible_households, maximum_donor_household_share=maximum_donor_household_share, ) + # The households and carriers seated, by id; ids differ across clones, so + # one map serves every clone. selected_household: dict[int, int] = {} carrier_person: dict[int, int] = {} band_eligible: dict[int, int] = {} @@ -588,43 +676,65 @@ def stack_spi_income_band_donors( count=len(candidates), ) eligible = (rate > 0.0) & ~taken_before - band_eligible[band] = int(pd.unique(candidate_households[eligible]).size) - if band_eligible[band] < seats[band]: + found = [ + int(pd.unique(candidate_households[positions[eligible[positions]]]).size) + for positions in clone_positions + ] + short = [count for count in found if count < seats[band]] + if short and clone_count == 1: raise ValueError( f"SPI income band donors need {seats[band]} candidate households " f"with positive weight and propensity for the band from {band}; " - f"found {band_eligible[band]}." + f"found {found[0]}." ) + if short: + raise ValueError( + f"SPI income band donors need {seats[band]} candidate households " + f"with positive weight and propensity for the band from {band} in " + f"each of the {clone_count} geographic clones; {len(short)} " + f"clone(s) found fewer, the fewest {min(short)}." + ) + band_eligible[band] = distinct_count(candidate_sources, eligible) uniforms = stable_identity_uniforms( candidate_ids, seed=seed, salt=f"{SPI_INCOME_BAND_DONOR_DRAW_SALT}:{band}", ) - # Efraimidis-Spirakis: the largest u ** (1 / rate) are a weighted draw - # without replacement; log(u) / rate orders them identically. - keys = np.full(len(candidates), -np.inf) - keys[eligible] = ( - np.log(np.maximum(uniforms[eligible], np.finfo(float).tiny)) - / rate[eligible] - ) - order = np.lexsort((candidate_ids, -keys)) - taken: set[int] = set() - for position in order: - if not eligible[position]: - break - household_id = int(candidate_households[position]) - if household_id in taken: - continue - taken.add(household_id) - selected_household[household_id] = band - carrier_person[int(candidate_ids[position])] = band - if len(taken) == seats[band]: - break - if len(taken) != seats[band]: - raise ValueError( - f"SPI income band donors could not seat {seats[band]} households " - f"for the band from {band}." + # The walk, once in each clone over its own candidates. + for positions in clone_positions: + clone_ids = candidate_ids[positions] + clone_households = candidate_households[positions] + clone_eligible = eligible[positions] + # Efraimidis-Spirakis: the largest u ** (1 / rate) are a weighted + # draw without replacement; log(u) / rate orders them identically. + keys = np.full(positions.size, -np.inf) + keys[clone_eligible] = ( + np.log( + np.maximum( + uniforms[positions][clone_eligible], np.finfo(float).tiny + ) + ) + / rate[positions][clone_eligible] ) + order = np.lexsort((clone_ids, -keys)) + taken: set[int] = set() + for position in order: + if not clone_eligible[position]: + break + household_id = int(clone_households[position]) + if household_id in taken: + continue + taken.add(household_id) + selected_household[household_id] = band + carrier_person[int(clone_ids[position])] = band + if len(taken) == seats[band]: + break + if len(taken) != seats[band]: + raise ValueError( + f"SPI income band donors could not seat {seats[band]} " + f"households for the band from {band}" + + ("." if clone_count == 1 else " in a geographic clone.") + ) selected_ids = set(selected_household) # The donors are copies of base households, minted at ordinal 2 of the @@ -681,9 +791,14 @@ def stack_spi_income_band_donors( # Funding: the donors' mass leaves the incumbent households of each donor's # stratum in proportion (the SPI support channel's own rule), so no mass is - # created and every stratum keeps its mass. + # created and every stratum keeps its mass. A clone's donors share the + # band's mass with every other clone's, and a stratum is a region within a + # clone, so every clone keeps its own regions' mass; the mass scale is the + # smallest any stratum allows, one for every clone, so each band keeps one + # donor weight. + clone_weight = {band: plan[band][1] / clone_count for band in bands} planned_weights = np.asarray( - [plan[int(value)][1] for value in donor_band], dtype=float + [clone_weight[int(value)] for value in donor_band], dtype=float ) incumbent_strata = _funding_strata(household) donor_strata = _funding_strata(donor_household) @@ -740,50 +855,65 @@ def stack_spi_income_band_donors( ) validate_uk_national_frame(result) + # The receipt counts donors and carriers in distinct source households + # and rounds them, withholding a mean resting on fewer than three (UKDS + # disclosure control, receipt_counts): a clone's copy of a household is + # the same survey household. + donor_sources = source_household_ids( + donor_household, stage=UK_SPI_INCOME_BAND_DONORS_STAGE_NAME + ) + by_donor = donor_household.set_index("household_id") carrier_rows = donor_person.loc[donor_person[PERSON_IS_SPI_INCOME_BAND_CARRIER]] carrier_region = ( + carrier_rows["person_household_id"].map(by_donor["region"]).astype(str) + ) + carrier_sources = ( carrier_rows["person_household_id"] - .map(donor_household.set_index("household_id")["region"]) - .astype(str) + .map(pd.Series(donor_sources, index=by_donor.index)) + .to_numpy() ) carrier_ages = pd.to_numeric(carrier_rows["age"], errors="coerce") band_of_carrier = carrier_rows["person_household_id"].map( - donor_household.set_index("household_id")[ - SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN - ] + by_donor[SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN] ) band_rows: list[dict[str, object]] = [] for band in bands: mask = donor_band == band carriers = band_of_carrier == band + carrier_support = distinct_count(carrier_sources, carriers.to_numpy()) band_rows.append( { "lower_bound": band, - "expected_donor_households": int(plan[band][0]), + "expected_donor_households": int(plan[band][0]) * clone_count, "donor_households": int(mask.sum()), "carriers": int(carriers.sum()), - "eligible_households": band_eligible[band], - "planned_donor_weight": float(plan[band][1]), - "donor_weight": float(plan[band][1] * mass_scale), + "eligible_households": rounded_count(band_eligible[band]), + "planned_donor_weight": float(clone_weight[band]), + "donor_weight": float(clone_weight[band] * mass_scale), "weighted_taxpayers": float(donor_weights[mask].sum()), "published_taxpayers": float(band_taxpayers[band]), - "carrier_mean_age": float(carrier_ages[carriers].mean()), + "carrier_mean_age": ( + float(carrier_ages[carriers].mean()) + if carrier_support >= UK_DISCLOSURE_FLOOR + else UK_WITHHELD + ), "carrier_regions": int(carrier_region[carriers].nunique()), } ) - donor_rows_by_stratum = ( - pd.Series(1, index=donor_strata.to_numpy()).groupby(level=0).sum() - ) + donor_stratum_labels = donor_strata.to_numpy() funding_rows = [ { "stratum": str(stratum), - "donor_households": int(donor_rows_by_stratum.get(stratum, 0)), + "donor_households": rounded_count( + distinct_count(donor_sources, donor_stratum_labels == stratum) + ), "incumbent_mass": float(incumbent_mass[stratum]), "donor_mass": float(donor_mass.get(stratum, 0.0) * mass_scale), "factor": float(factor_by_stratum[stratum]), } for stratum in sorted(incumbent_mass.index, key=str) ] + carrier_region_labels = carrier_region.to_numpy() return UKSPIIncomeBandDonorResult( frame=result, band_rows=tuple(band_rows), @@ -794,8 +924,9 @@ def stack_spi_income_band_donors( maximum_reallocation_share=float(maximum_reallocation_share), seating_scale=float(seating_scale), mass_scale=mass_scale, - eligible_households=eligible_households, - expected_donor_count=int(sum(count for count, _ in plan.values())), + eligible_households=rounded_count(eligible_households), + expected_donor_count=int(sum(count for count, _ in plan.values())) + * clone_count, donor_count=len(selected_household), reallocated_mass=float(donor_weights.sum()), old_total=old_total, @@ -804,15 +935,27 @@ def stack_spi_income_band_donors( sample_fraction=float(sample_fraction), taxpayer_period=int(taxpayer_period), carrier_region_counts={ - str(key): int(value) - for key, value in carrier_region.value_counts().sort_index().items() + region: rounded_count( + distinct_count(carrier_sources, carrier_region_labels == region) + ) + for region in sorted(set(carrier_region_labels)) }, propensity_cells=int(len(propensity)), + clone_count=clone_count, + funding_strata=( + *SPI_INCOME_BAND_FUNDING_STRATA, + *(() if clone_indices is None else (SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN,)), + ), ) def _funding_strata(household: pd.DataFrame) -> pd.Series: - """Each household's funding stratum label.""" + """Each household's funding stratum label. + + The declared strata joined by ``|``; where the table carries the + geographic clone index, the household's clone is appended, so every clone + funds its own donors from its own regions (microcosm#931). + """ missing = sorted(set(SPI_INCOME_BAND_FUNDING_STRATA) - set(household.columns)) if missing: @@ -820,12 +963,73 @@ def _funding_strata(household: pd.DataFrame) -> pd.Series: columns = [ household[column].astype(str) for column in SPI_INCOME_BAND_FUNDING_STRATA ] + if _geographic_clone_indices(household) is not None: + columns.append(household[SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN].astype(str)) label = columns[0] for column in columns[1:]: label = label + "|" + column return label.reset_index(drop=True) +def _geographic_clone_indices(household: pd.DataFrame) -> np.ndarray | None: + """Each household's geographic clone index, or None without the column. + + The clone-first spine (microcosm#931) gives every household the index of + its geographic clone in ``household_clone_index``, and its copies keep + their root's; a household table without the column, as on every spine + before ``uk.full.expand``, is one clone. Refuses a non-integer column, + a household without an index and a negative index, each by count or + dtype, never by id or value. + """ + + column = SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN + if column not in household.columns: + return None + values = household[column] + if pd.api.types.is_bool_dtype(values.dtype) or not ( + pd.api.types.is_integer_dtype(values.dtype) + ): + raise ValueError( + "SPI income band donors seat each geographic clone on an integer " + f"{column!r} column; it has dtype {values.dtype}." + ) + unindexed = int(values.isna().sum()) + if unindexed: + raise ValueError( + f"{unindexed} household row(s) carry no {column!r}; SPI income band " + "donors seat each geographic clone." + ) + indices = values.to_numpy(dtype=np.int64) + negative = int((indices < 0).sum()) + if negative: + raise ValueError( + f"{negative} household row(s) carry a negative {column!r}; SPI income " + "band donors seat each geographic clone." + ) + return indices + + +def _geographic_clone_count(indices: np.ndarray | None) -> int: + """``K``, one more than the largest clone index, refused with a gap below it. + + Every clone from 0 to ``K - 1`` must carry a household: each one seats + the band's seats, so a missing clone would leave its share of the + published mass unseated. The refusal counts the missing indices. + """ + + if indices is None or not indices.size: + return 1 + clone_count = int(indices.max()) + 1 + absent = clone_count - int(np.unique(indices).size) + if absent: + raise ValueError( + f"{absent} of the {clone_count} geographic clone indices from 0 to the " + "largest are carried by no household; SPI income band donors seat " + "clones 0 to K - 1." + ) + return clone_count + + #: Rows tried for the exact-total correction before the stage refuses. _EXACT_TOTAL_CORRECTION_CANDIDATES = 256 @@ -1046,6 +1250,7 @@ def _assert_band_donor_stage_parameters(stage: SourceStageSpec, *, seed: int) -> __all__ = [ "HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR", "PERSON_IS_SPI_INCOME_BAND_CARRIER", + "SPI_INCOME_BAND_CAPPED_SEATING", "SPI_INCOME_BAND_DONOR_CLONE_INDEX", "SPI_INCOME_BAND_DONOR_DRAW_SALT", "SPI_INCOME_BAND_DONOR_ID_REMAPPING", @@ -1054,6 +1259,7 @@ def _assert_band_donor_stage_parameters(stage: SourceStageSpec, *, seed: int) -> "SPI_INCOME_BAND_DONOR_MASS_CHANGE_REASON", "SPI_INCOME_BAND_DONOR_SEED", "SPI_INCOME_BAND_DONOR_TAPE_SEED", + "SPI_INCOME_BAND_FULL_SEATING", "SPI_INCOME_BAND_FUNDING_STRATA", "SPI_INCOME_BAND_MAXIMUM_DONOR_HOUSEHOLD_SHARE", "SPI_INCOME_BAND_MAXIMUM_DONOR_WEIGHT", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index 965fce8837..cd177c7372 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -1986,6 +1986,11 @@ def _cgt_support_split_gate( ) +#: The two forms the band donor receipt records its seating scale in: +#: ``spi_band_donors.SPI_INCOME_BAND_FULL_SEATING`` and ``..._CAPPED_SEATING``. +_BAND_DONOR_SEATINGS = ("full", "capped") + + def _spi_income_band_donor_support_gate( stage: str, evidence: Mapping[str, object], @@ -1996,9 +2001,16 @@ def _spi_income_band_donor_support_gate( The donors are a mass-conserving support channel (microcosm#1063): total household mass is unchanged, no incumbent stratum gives up more than the declared share, and no donor starts above the declared weight. On a full - frame (both scales one) each band seats its planned count, never fewer - than the minimum, and carries its published taxpayers; a build declaring - the full survey sample must be at full scale. + frame (full seating and a mass scale of one) each band seats its planned + count, never fewer than the minimum, and carries its published taxpayers; + a build declaring the full survey sample must be at full scale. + + On a frame of ``K`` geographic clones (the receipt's ``clone_count``, + microcosm#931) every clone seats each band's seats at the band's weight + over ``K``: a donor's weight times ``K`` must stay within the maximum, a + band's donors split evenly across the clones, and at full seating the + plan and the minimum are ``K`` times one clone's. The band's published + mass is checked as on one clone. """ check = "spi_income_band_donor_support" @@ -2029,21 +2041,43 @@ def _spi_income_band_donor_support_gate( failures.append( f"{stage}: receipt {key} {value} differs from the gate's {expected}." ) - scales: dict[str, float] = {} - for key in ("seating_scale", "mass_scale"): - scales[key] = _finite_number(evidence.get(key), label=f"{stage}.{key}") - if not 0.0 < scales[key] <= 1.0: - failures.append(f"{stage}: {key} {scales[key]} is outside (0, 1].") - full_seating = scales["seating_scale"] == 1.0 - full_scale = full_seating and scales["mass_scale"] == 1.0 + clone_count = evidence.get("clone_count") + if ( + not isinstance(clone_count, int) + or isinstance(clone_count, bool) + or clone_count < 1 + ): + raise ValueError(f"{stage}.clone_count must be a positive integer.") + over_clones = ( + "" + if clone_count == 1 + else f" divided by the geographic clone count {clone_count}" + ) + minimum_seated = minimum_donors * clone_count + per_clone_minimum = ( + "" + if clone_count == 1 + else f" ({minimum_donors} in each of the {clone_count} geographic clones)" + ) + # The receipt records the seating scale as full or capped: an exact scale + # below one gives back the eligible household count. + seating = evidence.get("seating_scale") + if seating not in _BAND_DONOR_SEATINGS: + failures.append( + f"{stage}: seating_scale {seating!r} is neither 'full' nor 'capped'." + ) + mass_scale = _finite_number(evidence.get("mass_scale"), label=f"{stage}.mass_scale") + if not 0.0 < mass_scale <= 1.0: + failures.append(f"{stage}: mass_scale {mass_scale} is outside (0, 1].") + full_seating = seating == "full" + full_scale = full_seating and mass_scale == 1.0 sample_fraction = _finite_number( evidence.get("sample_fraction"), label=f"{stage}.sample_fraction" ) if sample_fraction == 1.0 and not full_scale: failures.append( f"{stage}: a full-sample build must seat and fund the donors at full " - f"scale, got seating_scale {scales['seating_scale']} and mass_scale " - f"{scales['mass_scale']}." + f"scale, got seating_scale {seating} and mass_scale {mass_scale}." ) mass = _mapping(evidence.get("mass"), label=f"{stage}.mass") @@ -2104,16 +2138,23 @@ def _spi_income_band_donor_support_gate( failures.append( f"{stage}: band from {lower} donor weight {weight} is not positive." ) - if weight > maximum_donor_weight * (1.0 + _FLOAT_RELATIVE_TOLERANCE): + if clone_count * weight > maximum_donor_weight * ( + 1.0 + _FLOAT_RELATIVE_TOLERANCE + ): failures.append( f"{stage}: band from {lower} donor weight {weight} exceeds the " - f"maximum {maximum_donor_weight}." + f"maximum {maximum_donor_weight}{over_clones}." + ) + if donors % clone_count: + failures.append( + f"{stage}: band from {lower} seats {donors} donors, which do not " + f"split evenly across the {clone_count} geographic clones." ) - if full_seating and (donors != expected_donors or donors < minimum_donors): + if full_seating and (donors != expected_donors or donors < minimum_seated): failures.append( f"{stage}: band from {lower} seats {donors} donors at full scale; " f"the plan seats {expected_donors} and the minimum is " - f"{minimum_donors}." + f"{minimum_seated}{per_clone_minimum}." ) # A scaled frame seats fewer donors or lighter ones, so its bands fall # short of the published mass by construction; the receipt shows it. @@ -2168,8 +2209,9 @@ def _spi_income_band_donor_support_gate( details = { "bands_checked": len(bands), "donor_count": donor_count, - "seating_scale": scales["seating_scale"], - "mass_scale": scales["mass_scale"], + "clone_count": clone_count, + "seating_scale": seating, + "mass_scale": mass_scale, "reallocated_mass": reallocated, "relative_mass_deviation": deviation, "heaviest_donor_weight": heaviest_donor, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index eee420016d..3d4e02d6d1 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -10,16 +10,20 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.receipt_counts import UK_WITHHELD, rounded_count from microcosm.build.uk_runtime.spi_band_donors import ( HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, PERSON_IS_SPI_INCOME_BAND_CARRIER, + SPI_INCOME_BAND_CAPPED_SEATING, SPI_INCOME_BAND_DONOR_CLONE_INDEX, SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN, SPI_INCOME_BAND_DONOR_LOWER_BOUNDS, SPI_INCOME_BAND_DONOR_MASS_CHANGE_REASON, SPI_INCOME_BAND_DONOR_SEED, SPI_INCOME_BAND_DONOR_TAPE_SEED, + SPI_INCOME_BAND_FULL_SEATING, SPI_INCOME_BAND_MAXIMUM_DONOR_WEIGHT, SPI_INCOME_BAND_MINIMUM_DONORS, UKSPIIncomeBandDonorStageTransform, @@ -50,9 +54,11 @@ support_channel_column, support_clone_index_column, ) -from microcosm.frame import WeightKind +from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate +from microcosm.frame import WeightKind, engine_tables from test_support.microcosm_build.uk_clone_first_frames import ( assert_copy_ids_follow_lineage, + clone_first_frame, ) from test_support.paths import paths_for @@ -158,13 +164,15 @@ def _raw_tape(rows: int = 240, *, seed: int = 0) -> pd.DataFrame: return pd.DataFrame(records) -def _support_frame( +def _raw_support_tables( n_households: int = 60, *, reverse: bool = False, dependant_age: int = 12, dependant_is_claimant: bool = False, -): +) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + """The FRS base tables the support frames copy: a claimant and a dependant.""" + ids = np.arange(1, n_households + 1, dtype="int64") person = pd.DataFrame( { @@ -197,6 +205,64 @@ def _support_frame( "region": np.where(ids % 3 == 0, "SOUTH_EAST", "LONDON"), } ) + return person, benunit, household + + +def _support_frame( + n_households: int = 60, + *, + reverse: bool = False, + dependant_age: int = 12, + dependant_is_claimant: bool = False, + expected_spi_households: int = 10, +): + return _spi_support_frame( + *_raw_support_tables( + n_households, + reverse=reverse, + dependant_age=dependant_age, + dependant_is_claimant=dependant_is_claimant, + ), + expected_spi_households=expected_spi_households, + ) + + +def _clone_first_support_frame( + n_households: int, + n_clones: int, + *, + expected_spi_households: int = 10, + tables: tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame] | None = None, + **options, +): + """``_support_frame``'s households as geographic clones before stage 16. + + Clone ``k`` repeats every row at ``id + k x 10**9`` and an equal share of + its household's weight (``clone_first_frame``), and the SPI support + channel then copies each clone's households, as the clone-first spine will + (microcosm#931). ``tables`` replaces the raw tables ``options`` build. + """ + + person, benunit, household = ( + _raw_support_tables(n_households, **options) if tables is None else tables + ) + raw = uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2024", + weight_kind=WeightKind.DESIGN, + ) + cloned = engine_tables(clone_first_frame(raw, n_clones)) + return _spi_support_frame( + cloned["person"], + cloned["benunit"], + cloned["household"], + expected_spi_households=expected_spi_households, + ) + + +def _spi_support_frame(person, benunit, household, *, expected_spi_households: int): # The SPI copies are a keyed Bernoulli share at 10 / n (microcosm#931), so # their count is binomial. At seed 42 both regions keep copies at the 60 # and 200 households these tests build (11 copies, 2 in SOUTH_EAST, at @@ -205,7 +271,7 @@ def _support_frame( person, benunit, household, - expected_spi_households=10, + expected_spi_households=expected_spi_households, seed=42, source_year=2024, zero_weight_declarations=(), @@ -359,7 +425,9 @@ def test_stack_reserves_band_rows_and_conserves_every_region_mass() -> None: assert receipt.old_total == receipt.new_total == float(incoming.total) assert receipt.reason == SPI_INCOME_BAND_DONOR_MASS_CHANGE_REASON evidence = result.evidence() - assert evidence["seating_scale"] == 1.0 and evidence["mass_scale"] == 1.0 + assert result.seating_scale == 1.0 + assert evidence["seating_scale"] == "full" and evidence["mass_scale"] == 1.0 + assert evidence["clone_count"] == 1 assert evidence["reallocated_mass"] == pytest.approx(sum(SMALL_TAXPAYERS.values())) assert evidence["mass"]["old_total"] == evidence["mass"]["new_total"] @@ -426,7 +494,10 @@ def test_stack_scales_seats_then_mass_on_a_frame_too_small_for_the_plan() -> Non evidence = result.evidence() assert evidence["expected_donor_count"] == 1_920 assert evidence["donor_count"] == 9 - assert evidence["seating_scale"] == pytest.approx(0.15 * 60 / 1_920) + assert result.seating_scale == pytest.approx(0.15 * 60 / 1_920) + # The receipt says only that the cap bound: the exact scale gives back + # the eligible household count. + assert evidence["seating_scale"] == "capped" assert evidence["mass_scale"] == 1.0 assert float(result.frame.weights_for("household").total) == float( frame.weights_for("household").total @@ -959,9 +1030,19 @@ def test_stage_transform_runs_on_a_synthetic_tape_and_receipts() -> None: # The survey-side sample fraction is provenance only: the seats follow # the frame, not the fraction. assert evidence["sample_fraction"] == 0.01 - assert evidence["donor_count"] == 9 and evidence["seating_scale"] < 1.0 + assert evidence["donor_count"] == 9 and evidence["seating_scale"] == "capped" assert [row["lower_bound"] for row in evidence["bands"]] == list(BANDS) - assert sum(evidence["carrier_region_counts"].values()) == 9 + # The carriers' regions, each count rounded on distinct source households. + household = result.table("household") + carriers = result.table("person").loc[ + lambda people: people[PERSON_IS_SPI_INCOME_BAND_CARRIER] + ] + region_of = household.set_index("household_id")["region"] + per_region = carriers["person_household_id"].map(region_of).value_counts() + assert int(per_region.sum()) == 9 + assert evidence["carrier_region_counts"] == { + str(region): rounded_count(int(count)) for region, count in per_region.items() + } assert evidence["draw"] == { "seed": SPI_INCOME_BAND_DONOR_SEED, "salt": "spi_income_band_donor_draw", @@ -1038,7 +1119,7 @@ def test_e8_receipt_reconstructs_the_seating_from_ids() -> None: assert problems == {} assert receipt["seating_recomputed"] is True assert receipt["donor_households"] == 27 - assert 0.0 < receipt["seating_scale"] < 1.0 + assert receipt["seating_scale"] == "capped" assert receipt["mass"]["declared_factor"] == 1.0 assert [row["lower_bound"] for row in receipt["bands"]] == list(BANDS) assert set(receipt["funding"]) == {"LONDON", "SOUTH_EAST"} @@ -1104,3 +1185,620 @@ def test_e8_receipt_reports_a_seating_the_ids_do_not_reproduce() -> None: assert problems["band_donor_mass_record"] == "missing" with pytest.raises(ValueError, match="donor layer is absent"): tool._e8_band_donors(frame, {}, **arguments) + + +# Seating per geographic clone (microcosm#931). The clone-first spine gives +# every household its clone; each clone seats its own donors, by the same walk +# and the same seats, and funds them from its own regions. + +_CLONE = "household_clone_index" +_PUBLISHED = {"band_taxpayers": TAXPAYERS, "taxpayer_period": 2024} +#: The stage gate's declaration with the small frames' seating constants. +_SMALL_GATE_PARAMETERS = { + "stage": "spi_income_band_donors", + "check": "spi_income_band_donor_support", + "minimum_donors_per_band": 3, + "maximum_donor_weight": 10.0, + "minimum_funding_factor": 0.5, + "maximum_band_taxpayer_deviation": 0.5, + "maximum_relative_mass_deviation": 1e-9, + "band_lower_bounds": list(BANDS), +} +#: ``uk_frame_content_identity`` of the frames stage 17 reads and writes below, +#: computed with the stage at 50b6e629b, before it seated per geographic clone +#: (microcosm#931). On one clone the stage keeps every value, weight and mass +#: record, so the identities hold. Identities compare within the pinned +#: environment only (``content_identity``). +_ONE_CLONE_STAGE_17_INPUT_IDENTITIES = { + 200: "129e44480d4409444baf231a25477d50a8c2dbcb2e525af5f13a53a353908e50", + 60: "13ab2b40acb59e04cfd902f83dc0f0bdc69924adbc0c834315ac5610fc404ddd", +} +_ONE_CLONE_STAGE_17_IDENTITIES = { + "full_seating": "d804531b98002638548b29c3c9e2dbb7a9363e19a079d502dab9323816ebae8f", + "capped_seating": "2f8afbee4d45fea6c1787c4b183243bc49f260c9c490de06bddb238db3db7683", + "scaled_mass": "5751c98409fa2e2d382ba35eb114ac1c10b2ccad59d0ac9de7c7db7840233a08", +} + + +def _clone_of(result) -> pd.Series: + """Each household id's geographic clone, 0 on a frame without the index.""" + + household = result.frame.table("household") + clones = ( + household[_CLONE].to_numpy() + if _CLONE in household.columns + else np.zeros(len(household), dtype=np.int64) + ) + return pd.Series(clones, index=household["household_id"].to_numpy()) + + +def _seating(result) -> dict[int, dict[int, float]]: + """Each clone's seats: the donor's true source household -> its band.""" + + household = result.frame.table("household") + donors = household.loc[household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR]] + clones = donors["household_id"].map(_clone_of(result)).to_numpy() + return { + int(clone): dict( + zip( + rows["source_household_id"].astype(int), + rows[SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN], + strict=True, + ) + ) + for clone, rows in donors.groupby(clones) + } + + +def _carriers(result) -> dict[int, set[int]]: + """Each clone's carriers, by the person id each copy was made from.""" + + person = result.frame.table("person") + carriers = person.loc[person[PERSON_IS_SPI_INCOME_BAND_CARRIER]] + clones = carriers["person_household_id"].map(_clone_of(result)).to_numpy() + return { + int(clone): set(rows["person_source_id"].astype(int)) + for clone, rows in carriers.groupby(clones) + } + + +def _weights_by_id(frame) -> pd.Series: + return pd.Series( + np.asarray(frame.weights_for("household").values, dtype=float), + index=frame.table("household")["household_id"].to_numpy(), + ) + + +def _mass_by_clone_and_region(frame) -> dict[tuple[int, str], float]: + household = frame.table("household") + weights = np.asarray(frame.weights_for("household").values, dtype=float) + return { + (int(clone), str(region)): float(mass) + for (clone, region), mass in pd.Series(weights) + .groupby([household[_CLONE].to_numpy(), household["region"].to_numpy()]) + .sum() + .items() + } + + +def _with_tables(frame, **tables): + """``frame`` with tables replaced; its weights and mass records kept.""" + + weights = frame.weights_for("household") + return uk_national_frame( + person=tables.get("person", frame.table("person")), + benunit=tables.get("benunit", frame.table("benunit")), + household=tables.get("household", frame.table("household")), + time_period="2024", + weight_kind=weights.kind, + household_weights=weights.values, + mass_log=frame.mass_log, + ) + + +def _small_gate(evidence): + return uk_stage_health_gate( + evidence=evidence, + stage="spi_income_band_donors", + check="spi_income_band_donor_support", + parameters=_SMALL_GATE_PARAMETERS, + ) + + +def test_one_clone_frames_keep_their_content_identity_through_stage_17() -> None: + support = {200: _support_frame(200), 60: _support_frame()} + for households, frame in support.items(): + assert ( + uk_frame_content_identity(frame) + == _ONE_CLONE_STAGE_17_INPUT_IDENTITIES[households] + ), "the synthetic input moved: re-pin both tables, recording what moved it" + results = { + "full_seating": _stack_small(support[200]), + "capped_seating": stack_spi_income_band_donors( + support[60], propensity=_propensity(), **_PUBLISHED + ), + "scaled_mass": _stack_small( + support[60], + band_taxpayers={ + band: value * 400.0 for band, value in SMALL_TAXPAYERS.items() + }, + maximum_donor_weight=4_000.0, + ), + } + assert results["scaled_mass"].mass_scale < 1.0 + assert results["capped_seating"].seating_scale < 1.0 + assert { + name: uk_frame_content_identity(result.frame) + for name, result in results.items() + } == _ONE_CLONE_STAGE_17_IDENTITIES + + +def test_a_one_clone_index_seats_weighs_and_funds_bit_for_bit_as_no_index() -> None: + plain_frame = _support_frame(200) + household = plain_frame.table("household") + indexed_frame = _with_tables( + plain_frame, + household=household.assign(**{_CLONE: np.zeros(len(household), "int64")}), + ) + plain, indexed = _stack_small(plain_frame), _stack_small(indexed_frame) + + assert _seating(indexed) == _seating(plain) + assert _carriers(indexed) == _carriers(plain) + np.testing.assert_array_equal( + indexed.frame.weights_for("household").values, + plain.frame.weights_for("household").values, + ) + assert indexed.frame.mass_log == plain.frame.mass_log + # The funding strata gain the clone in their labels and nothing else. + assert [row["factor"] for row in indexed.funding_rows] == [ + row["factor"] for row in plain.funding_rows + ] + assert [row["stratum"] for row in indexed.funding_rows] == [ + f"{row['stratum']}|0" for row in plain.funding_rows + ] + assert plain.evidence()["funding_strata"] == ["region"] + assert indexed.evidence()["funding_strata"] == ["region", _CLONE] + assert indexed.evidence()["clone_count"] == plain.evidence()["clone_count"] == 1 + + +@pytest.mark.parametrize("n_clones", [3, 7]) +def test_clone_zero_seats_what_one_clone_seats_and_every_clone_takes_the_seats( + n_clones: int, +) -> None: + # Given the same households as one clone and as clones made before the + # SPI support channel, + one = _stack_small(_support_frame(200)) + many = _stack_small(_clone_first_support_frame(200, n_clones)) + seated, seated_once = _seating(many), _seating(one)[0] + + # then clone 0 seats exactly the one clone's households, bands and + # carriers, every clone seats the same number in each band, and every + # other clone draws households of its own. + assert sorted(seated) == list(range(n_clones)) + assert seated[0] == seated_once + assert _carriers(many)[0] == _carriers(one)[0] + seats = pd.Series(seated_once).value_counts().to_dict() + for clone in range(n_clones): + assert pd.Series(seated[clone]).value_counts().to_dict() == seats + for clone in range(1, n_clones): + assert set(seated[clone]) != set(seated[0]) + + # Clone 0's weights, its donors' included, are the one clone's over the + # clone count, to the exact-total corrections clone 0 absorbs. + weights_once = _weights_by_id(one.frame) + clone_zero = _clone_of(many).loc[lambda clones: clones.eq(0)].index + assert set(clone_zero) == set(weights_once.index) + np.testing.assert_allclose( + _weights_by_id(many.frame).loc[clone_zero].to_numpy() * n_clones, + weights_once.loc[clone_zero].to_numpy(), + rtol=1e-12, + ) + rows_once = {row["lower_bound"]: row for row in one.band_rows} + for row in many.band_rows: + once = rows_once[row["lower_bound"]] + assert row["donor_weight"] * n_clones == pytest.approx( + once["donor_weight"], rel=1e-12 + ) + assert row["weighted_taxpayers"] == pytest.approx( + once["weighted_taxpayers"], rel=1e-12 + ) + for key in ("expected_donor_households", "donor_households", "carriers"): + assert row[key] == n_clones * once[key] + factors_once = {row["stratum"]: row["factor"] for row in one.funding_rows} + factors = {row["stratum"]: row["factor"] for row in many.funding_rows} + assert set(factors) == { + f"{region}|{clone}" for region in factors_once for clone in range(n_clones) + } + for region, factor in factors_once.items(): + assert factors[f"{region}|0"] == pytest.approx(factor, rel=1e-12) + evidence = many.evidence() + assert evidence["clone_count"] == n_clones + assert evidence["mass_scale"] == one.mass_scale == 1.0 + assert evidence["donor_count"] == n_clones * one.evidence()["donor_count"] + assert evidence["expected_donor_count"] == n_clones * 13 + + +def test_the_seats_count_source_households_so_the_cap_binds_alike_at_any_k() -> None: + # Given a frame too small for the published plan, the household share's + # cap binds. Counted in rows, three clones would treble the eligible + # households and lift the cap; counted in source households, they do not. + propensity = _propensity() + one = stack_spi_income_band_donors( + _support_frame(), propensity=propensity, **_PUBLISHED + ) + many = stack_spi_income_band_donors( + _clone_first_support_frame(60, 3), propensity=propensity, **_PUBLISHED + ) + assert 0.0 < one.seating_scale < 1.0 + assert many.seating_scale == one.seating_scale + assert many.evidence()["seating_scale"] == "capped" + assert many.evidence()["eligible_households"] == rounded_count(60) + assert one.evidence()["eligible_households"] == rounded_count(60) + seats = pd.Series(_seating(one)[0]).value_counts().to_dict() + for seated in _seating(many).values(): + assert pd.Series(seated).value_counts().to_dict() == seats + + +def test_each_clone_funds_its_donors_from_its_own_regions() -> None: + frame = _clone_first_support_frame(200, 3) + result = _stack_small(frame) + assert _mass_by_clone_and_region(result.frame) == pytest.approx( + _mass_by_clone_and_region(frame), rel=1e-12 + ) + # A band's donors weigh one weight in every clone. + household = result.frame.table("household") + weights = _weights_by_id(result.frame) + donors = household.loc[household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR]] + for band, rows in donors.groupby(SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN): + assert weights.loc[rows["household_id"]].nunique() == 1, band + # Each clone and region gives up its own donors' mass. + donor_mass = ( + weights.loc[donors["household_id"]] + .groupby( + [ + donors[_CLONE].to_numpy(), + donors["region"].to_numpy(), + ] + ) + .sum() + ) + for row in result.funding_rows: + region, clone = row["stratum"].split("|") + assert row["donor_mass"] == pytest.approx( + float(donor_mass.get((int(clone), region), 0.0)), rel=1e-12 + ) + assert row["factor"] == pytest.approx( + 1.0 - row["donor_mass"] / row["incumbent_mass"], rel=1e-12 + ) + + +def test_one_mass_scale_serves_every_clone_when_one_clone_region_binds() -> None: + # Given clone 1 at a five-hundredth of its weight, its regions cannot fund + # their donors at the band weights, + frame = _clone_first_support_frame(200, 2) + household = frame.table("household") + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + weights[household[_CLONE].eq(1).to_numpy()] *= 0.002 + light = uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=household, + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=weights, + ) + result = _stack_small(light, sample_fraction=0.01) + evidence = result.evidence() + + # then the binding stratum gives up exactly the declared share, and its + # scale serves every band in every clone, so each band keeps one weight. + factors = {row["stratum"]: row["factor"] for row in evidence["funding"]} + binding = min(factors, key=factors.get) + assert 0.0 < evidence["mass_scale"] < 1.0 + assert binding.endswith("|1") + assert factors[binding] == pytest.approx(0.5, rel=1e-12) + assert all(factor > 0.5 for name, factor in factors.items() if name != binding) + out = result.frame.table("household") + donor_weights = _weights_by_id(result.frame).loc[ + out.loc[out[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR], "household_id"] + ] + bands = out.loc[ + out[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR], + SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN, + ].to_numpy() + for row in evidence["bands"]: + band_weights = donor_weights.to_numpy()[bands == row["lower_bound"]] + assert np.unique(band_weights).size == 1 + assert band_weights[0] == row["donor_weight"] + assert row["donor_weight"] == pytest.approx( + row["planned_donor_weight"] * evidence["mass_scale"], rel=1e-12 + ) + assert _mass_by_clone_and_region(result.frame) == pytest.approx( + _mass_by_clone_and_region(light), rel=1e-12 + ) + # The stage gate reconciles the scaled receipt band by band. + gate = _small_gate(evidence) + assert gate.passed, gate.failures + assert gate.details["smallest_funding_factor"] == pytest.approx(0.5, rel=1e-12) + + +def test_a_clone_without_candidates_is_refused_by_count() -> None: + frame = _clone_first_support_frame(200, 2) + person = frame.table("person").copy() + clone_of = frame.table("household").set_index("household_id")[_CLONE] + in_clone_one = person["person_household_id"].map(clone_of).eq(1).to_numpy() + person.loc[in_clone_one, "is_uc_claimant"] = False + + with pytest.raises(ValueError) as caught: + _stack_small(_with_tables(frame, person=person)) + assert str(caught.value) == ( + "SPI income band donors need 3 candidate households with positive weight " + "and propensity for the band from 2000000 in each of the 2 geographic " + "clones; 1 clone(s) found fewer, the fewest 0." + ) + + +@pytest.mark.parametrize( + ("index", "message"), + [ + ( + lambda clones: clones.astype(float), + "SPI income band donors seat each geographic clone on an integer " + "'household_clone_index' column; it has dtype float64.", + ), + ( + lambda clones: pd.array( + [None, *clones.to_numpy()[1:].tolist()], dtype="Int64" + ), + "1 household row(s) carry no 'household_clone_index'; SPI income band " + "donors seat each geographic clone.", + ), + ( + lambda clones: np.r_[[-1, -1], clones.to_numpy()[2:]], + "2 household row(s) carry a negative 'household_clone_index'; SPI " + "income band donors seat each geographic clone.", + ), + ( + lambda clones: clones.replace({1: 2}).to_numpy(), + "1 of the 3 geographic clone indices from 0 to the largest are carried " + "by no household; SPI income band donors seat clones 0 to K - 1.", + ), + ], +) +def test_a_malformed_clone_index_is_refused_by_count(index, message: str) -> None: + frame = _clone_first_support_frame(60, 2) + household = frame.table("household").copy() + household[_CLONE] = index(household[_CLONE]) + with pytest.raises(ValueError) as caught: + _stack_small(_with_tables(frame, household=household)) + assert str(caught.value) == message + + +def test_the_clone_seating_is_keyed_on_identity_in_any_row_order() -> None: + first = _stack_small(_clone_first_support_frame(200, 3)) + second = _stack_small(_clone_first_support_frame(200, 3, reverse=True)) + assert _seating(first) == _seating(second) + assert _carriers(first) == _carriers(second) + + +def test_the_receipt_counts_source_households_at_the_disclosure_floor() -> None: + # Given 200 households, 7 of them without an adult claimant, + person, benunit, household = _raw_support_tables(200) + person.loc[person["person_household_id"].le(7), "age"] = 12 + result = _stack_small( + _spi_support_frame(person, benunit, household, expected_spi_households=10) + ) + evidence = result.evidence() + + # then every count of survey households is rounded: the eligible ones in + # all and per band, the donors per funding stratum and the carriers per + # region, with no exact count beside them. + assert evidence["eligible_households"] == rounded_count(193) == 190 + propensity = _propensity() + claimants = person.loc[person["is_uc_claimant"] & person["age"].ge(16)] + region_of = household.set_index("household_id")["region"] + cells = { + int(row.person_household_id): ( + region_of[row.person_household_id], + str(row.gender), + int(age_band_lower_bound(np.asarray([float(row.age)]))[0]), + ) + for row in claimants.itertuples() + } + seated = _seating(result)[0] + taken: set[int] = set() + rows = {row["lower_bound"]: row for row in evidence["bands"]} + for band in sorted(BANDS, reverse=True): + positive = propensity.loc[ + (propensity["band"] == band) & (propensity["propensity"] > 0.0) + ] + open_cells = set( + zip( + positive["region"], + positive["gender"], + positive["age_band"], + strict=True, + ) + ) + eligible = { + household_id + for household_id, cell in cells.items() + if cell in open_cells and household_id not in taken + } + assert rows[band]["eligible_households"] == rounded_count(len(eligible)) + taken |= {source for source, seat in seated.items() if seat == band} + out = result.frame.table("household") + donors = out.loc[out[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR]] + by_region = donors["region"].value_counts() + assert {row["stratum"]: row["donor_households"] for row in evidence["funding"]} == { + str(region): rounded_count(int(by_region.get(region, 0))) + for region in ("LONDON", "SOUTH_EAST") + } + assert evidence["carrier_region_counts"] == { + str(region): rounded_count(int(count)) for region, count in by_region.items() + } + assert evidence["seating_scale"] == "full" + assert all(isinstance(row["carrier_mean_age"], float) for row in evidence["bands"]) + assert set(evidence) == { + "stage", + "minimum_donors_per_band", + "maximum_donor_weight", + "maximum_donor_household_share", + "maximum_reallocation_share", + "clone_count", + "funding_strata", + "seating_scale", + "mass_scale", + "sample_fraction", + "eligible_households", + "expected_donor_count", + "donor_count", + "reallocated_mass", + "mass", + "draw", + "taxpayer_count_resource", + "taxpayer_count_period", + "propensity", + "propensity_cells", + "bands", + "funding", + "carrier_region_counts", + } + assert {key for row in evidence["bands"] for key in row} == { + "lower_bound", + "expected_donor_households", + "donor_households", + "carriers", + "eligible_households", + "planned_donor_weight", + "donor_weight", + "weighted_taxpayers", + "published_taxpayers", + "carrier_mean_age", + "carrier_regions", + } + assert {key for row in evidence["funding"] for key in row} == { + "stratum", + "donor_households", + "incumbent_mass", + "donor_mass", + "factor", + } + + # A band whose carriers come from fewer than three households withholds + # their mean age: the published plan on 60 households seats one in three + # of the bands. + capped = stack_spi_income_band_donors( + _support_frame(), propensity=propensity, **_PUBLISHED + ).evidence() + for row in capped["bands"]: + if row["carriers"] < 3: + assert row["carrier_mean_age"] == UK_WITHHELD + else: + assert isinstance(row["carrier_mean_age"], float) + assert sum(row["carriers"] < 3 for row in capped["bands"]) == 3 + + +def test_a_mean_age_resting_on_fewer_than_three_source_households_is_withheld() -> None: + # Given one claimant alone the tape lets carry the top band, copied into + # each of three clones, + person, benunit, household = _raw_support_tables(200) + elder = person["person_household_id"].eq(1) & person["is_uc_claimant"] + person.loc[elder, "age"] = 70 + cell = ( + household.set_index("household_id").loc[1, "region"], + str(person.loc[elder, "gender"].iloc[0]), + int(age_band_lower_bound(np.asarray([70.0]))[0]), + ) + propensity = _propensity() + propensity = pd.concat( + [ + propensity.loc[propensity["band"] != 2_000_000], + pd.DataFrame( + [ + { + "region": cell[0], + "gender": cell[1], + "age_band": cell[2], + "band": 2_000_000, + "band_weight": 1.0, + "cell_weight": 1.0, + "propensity": 1.0, + } + ] + ), + ], + ignore_index=True, + ) + frame = _clone_first_support_frame(200, 3, tables=(person, benunit, household)) + result = _stack_small(frame, propensity=propensity, minimum_donors=1) + + # then each clone seats that household, and the band's three carriers + # rest on one source household: their mean age is withheld. + seated = _seating(result) + assert sorted(seated) == [0, 1, 2] + assert all(seats[1] == 2_000_000 for seats in seated.values()) + top = {row["lower_bound"]: row for row in result.band_rows}[2_000_000] + assert top["carriers"] == 3 + assert top["carrier_mean_age"] == UK_WITHHELD + first = {row["lower_bound"]: row for row in result.band_rows}[200_000] + assert first["carriers"] == 12 and isinstance(first["carrier_mean_age"], float) + + +def test_the_stage_gate_reads_a_receipt_by_its_clone_count() -> None: + one = _stack_small(_support_frame(200)).evidence() + evidence = _stack_small(_clone_first_support_frame(200, 3)).evidence() + for receipt in (one, evidence): + gate = _small_gate(receipt) + assert gate.passed, gate.failures + assert _small_gate(evidence).details["clone_count"] == 3 + + # Dropped, the clone count is refused rather than read as one clone. + with pytest.raises(ValueError, match="clone_count must be a positive integer"): + _small_gate( + {key: value for key, value in evidence.items() if key != "clone_count"} + ) + # Misread, three clones' donors split unevenly or outweigh the maximum. + failures = " ".join(_small_gate({**evidence, "clone_count": 2}).failures) + assert "do not split evenly across the 2 geographic clones" in failures + failures = " ".join(_small_gate({**evidence, "clone_count": 4}).failures) + assert "divided by the geographic clone count 4" in failures + # The gate spells the receipt's two seating forms itself. + from microcosm.build.uk_runtime import stage_health + + assert stage_health._BAND_DONOR_SEATINGS == ( + SPI_INCOME_BAND_FULL_SEATING, + SPI_INCOME_BAND_CAPPED_SEATING, + ) + + +def test_e8_receipt_replays_the_seating_of_every_clone() -> None: + tool = _identity_tool() + propensity = _propensity() + arguments = { + "propensity": propensity, + "band_taxpayers": TAXPAYERS, + "permutation_seed": 123, + } + stacked = stack_spi_income_band_donors( + _clone_first_support_frame(200, 3), propensity=propensity, **_PUBLISHED + ).frame + problems: dict[str, object] = {} + receipt = tool._e8_band_donors(stacked, problems, **arguments) + assert problems == {} + assert receipt["seating_recomputed"] is True + assert receipt["donor_households"] == 3 * 27 + assert receipt["seating_scale"] == "capped" + assert set(receipt["funding"]) == { + f"{region}|{clone}" for region in ("LONDON", "SOUTH_EAST") for clone in range(3) + } + + # A donor reweighted in the last clone breaks its band's one weight. + household = stacked.table("household") + weights = np.asarray(stacked.weights_for("household").values, dtype=float).copy() + last_donor = int(np.flatnonzero(household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR])[-1]) + assert household[_CLONE].iloc[last_donor] == 2 + weights[last_donor] *= 1.5 + problems = {} + tool._e8_band_donors(_reweighted(stacked, weights), problems, **arguments) + assert problems["band_donor_layer"] == {"bands_with_unequal_donor_weights": 1} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py index eebc06790e..62afee24ba 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py @@ -2191,7 +2191,8 @@ def _spi_income_band_donor_evidence() -> dict[str, object]: "stage": "spi_income_band_donors", "minimum_donors_per_band": 120, "maximum_donor_weight": 250.0, - "seating_scale": 1.0, + "clone_count": 1, + "seating_scale": "full", "mass_scale": 1.0, "sample_fraction": 1.0, "donor_count": 1_920, @@ -2325,7 +2326,7 @@ def test_spi_income_band_donor_support_gate_scales_only_off_the_full_sample() -> donor_mass = sum(row["donor_weight"] * 2 for row in rows) scaled = { **evidence, - "seating_scale": 8 / 1_920, + "seating_scale": "capped", "mass_scale": scale, "sample_fraction": 0.01, "donor_count": 8, @@ -2346,8 +2347,100 @@ def test_spi_income_band_donor_support_gate_scales_only_off_the_full_sample() -> # funds the donors at full scale. result = _spi_income_band_donor_gate({**scaled, "sample_fraction": 1.0}) assert not result.passed and "at full scale" in " ".join(result.failures) + assert ( + "spi_income_band_donors: a full-sample build must seat and fund the donors " + "at full scale, got seating_scale capped and mass_scale 0.5." + ) in result.failures result = _spi_income_band_donor_gate({**scaled, "mass_scale": 0.0}) assert not result.passed and "outside (0, 1]" in " ".join(result.failures) + # The receipt records the seating scale as full or capped, never as the + # exact scale, which gives back the eligible household count. + result = _spi_income_band_donor_gate({**scaled, "seating_scale": 8 / 1_920}) + assert not result.passed + assert ( + "spi_income_band_donors: seating_scale 0.004166666666666667 is neither " + "'full' nor 'capped'." + ) in result.failures + + +def _three_clone_band_donor_evidence() -> dict[str, object]: + """The full-scale receipt seated in each of three geographic clones. + + Every clone seats each band's plan at the band's weight over three, so + the bands carry the same published mass as on one clone (microcosm#931). + """ + + evidence = _spi_income_band_donor_evidence() + rows = [ + { + **row, + "expected_donor_households": 3 * row["expected_donor_households"], + "donor_households": 3 * row["donor_households"], + "carriers": 3 * row["carriers"], + "planned_donor_weight": row["planned_donor_weight"] / 3, + "donor_weight": row["donor_weight"] / 3, + } + for row in evidence["bands"] + ] + return {**evidence, "clone_count": 3, "donor_count": 3 * 1_920, "bands": rows} + + +def test_spi_income_band_donor_support_gate_scales_its_checks_by_the_clone_count() -> ( + None +): + evidence = _three_clone_band_donor_evidence() + result = _spi_income_band_donor_gate(evidence) + assert _passed(result), result.failures + assert result.details["clone_count"] == 3 + assert result.details["heaviest_donor_weight"] == pytest.approx(250.0 / 3) + + # A donor whose weight times the clone count exceeds the maximum fails. + heavy = [dict(row) for row in evidence["bands"]] + heavy[0]["donor_weight"] = 100.0 + assert ( + "spi_income_band_donors: band from 200000 donor weight 100.0 exceeds the " + "maximum 250.0 divided by the geographic clone count 3." + ) in _spi_income_band_donor_gate({**evidence, "bands": heavy}).failures + # A band whose donors do not split evenly across the clones fails, as does + # one seated below three clones' minimum at full scale. + uneven = [dict(row) for row in evidence["bands"]] + uneven[1].update(donor_households=733, carriers=733) + failures = _spi_income_band_donor_gate({**evidence, "bands": uneven}).failures + assert ( + "spi_income_band_donors: band from 500000 seats 733 donors, which do not " + "split evenly across the 3 geographic clones." + ) in failures + short = [dict(row) for row in evidence["bands"]] + short[3].update(expected_donor_households=357, donor_households=357, carriers=357) + assert ( + "spi_income_band_donors: band from 2000000 seats 357 donors at full scale; " + "the plan seats 357 and the minimum is 360 (120 in each of the 3 " + "geographic clones)." + ) in _spi_income_band_donor_gate({**evidence, "bands": short}).failures + # The same rows read as one clone's: no row is checked against three + # clones, and the one-clone messages are as they were. + one = _spi_income_band_donor_evidence() + heavy = [dict(row) for row in one["bands"]] + heavy[0].update(donor_households=718, carriers=718, donor_weight=500.0) + failures = _spi_income_band_donor_gate({**one, "bands": heavy}).failures + assert ( + "spi_income_band_donors: band from 200000 donor weight 500.0 exceeds the " + "maximum 250.0." + ) in failures + assert ( + "spi_income_band_donors: band from 200000 seats 718 donors at full scale; " + "the plan seats 1436 and the minimum is 120." + ) in failures + + # The clone count must be a positive integer the receipt states. + for value in (None, 0, -1, 1.0, True, "3"): + broken = dict(evidence) + if value is None: + del broken["clone_count"] + else: + broken["clone_count"] = value + with pytest.raises(ValueError, match="clone_count must be a positive integer"): + _spi_income_band_donor_gate(broken) def test_spi_support_channel_checks_the_declared_pension_age_share() -> None: diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 1fc680e4fe..71b19df8b2 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -7395,7 +7395,7 @@ "nonnegative_outputs": [ "spi_income_band_donor_lower_bound" ], - "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves).", + "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, with the seats counted in distinct source households, each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves).", "operations": [ { "band_column": "spi_income_band_donor_lower_bound", @@ -7412,16 +7412,16 @@ "conservation": "exact_total", "count_rule": "max(minimum_donors_per_band, ceil(published band taxpayers / maximum_donor_weight))", "declared_factor": 1, - "draw": "weighted_without_replacement_identity_keyed: Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", + "draw": "weighted_without_replacement_identity_keyed within each geographic clone (one before uk.full.expand): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", "expected_band_count": 4, "flag_column": "household_is_spi_income_band_donor", - "funding": "stratum reallocation: every incumbent household of the stratum a donor's source household belongs to is scaled by 1 - donor mass / stratum mass", + "funding": "stratum reallocation per geographic clone: every incumbent household of the stratum a donor's source household belongs to, in the donor's clone, is scaled by 1 - donor mass / stratum mass", "funding_strata": [ "region" ], "id_remapping": "copy-id table field spi_support_channel; a donor takes every entity id + 2 x the field's radix (10**11), and its source is read from its source-id columns, never from the id", "kind": "stack_income_band_donor_households", - "mass_scale": "min(1, min over funding strata of maximum_reallocation_share x stratum mass / seated donor mass); every band's weight scaled alike", + "mass_scale": "min(1, min over the funding strata of every geographic clone of maximum_reallocation_share x stratum mass / seated donor mass); one scale for every band and clone", "maximum_donor_household_share": 0.15, "maximum_donor_weight": 250, "maximum_reallocation_share": 0.5, @@ -7430,7 +7430,7 @@ "propensity": "SPI 2022-23 FACT-weighted band share by region, sex and age band", "reason": "Reallocate the published HMRC Table 2.5 taxpayer mass of each total-income band from GBP 200,000 from the households of each donor's region to its income band donor support rows; every region's household mass and the total national mass are conserved.", "salt": "spi_income_band_donor_draw", - "seating_scale": "min(1, maximum_donor_household_share x eligible base households / expected donor count); band counts floored, at least one per band", + "seating_scale": "min(1, maximum_donor_household_share x eligible base source households / expected donor count of one geographic clone); band counts floored, at least one per band", "seed": 3, "tape_preparation": "as hmrc_spi_income_spine prepares the tape: its stage-1 seed and draw_spi_donor_ages_by_population", "tape_preparation_seed": 42, @@ -7438,7 +7438,7 @@ "taxpayer_count_period": "build_period_tax_year", "taxpayer_count_resource": "hmrc_itl_taxpayer_counts.json", "weight_kind_out": "importance", - "weight_rule": "published band taxpayers / band donor count, equal within a band" + "weight_rule": "published band taxpayers / (geographic clone count x band donor count), equal within a band" } ], "outputs": [ diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 80b081846c..62f4640246 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"85696cd009ce349a132f753c55cb2168ed2f548de9f628ce74a71e99a0e9e6ba","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"fec5bddc81b70da31dcf9a301405049b71b11113bcdd2c2c5950efc8e3a4dc18","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"fec5bddc81b70da31dcf9a301405049b71b11113bcdd2c2c5950efc8e3a4dc18","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index e5e7a84ace..ca32649fcd 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -1310,10 +1310,12 @@ def _e8_band_donors( The donors are a funded support channel: each band's donors carry equal weights and their mass left the incumbent households of each donor's - region in proportion. Folding the CGT layers stacked afterwards gives the - frame the stage wrote; scaling each region's incumbents back to the - region's whole mass gives the weights it read. The stored layer must - agree with itself (one carrier per donor, each source household copied + region in proportion; where the frame carries geographic clones, the + region within the donor's clone, and each clone seats its own donors + (microcosm#931). Folding the CGT layers stacked afterwards gives the + frame the stage wrote; scaling each funding stratum's incumbents back to + the stratum's whole mass gives the weights it read. The stored layer must + agree with itself (one carrier per donor, each parent household copied once, equal weights within a band, the mass record conserving the total) and, when the tape's band propensities are supplied (they need the licensed SPI tape), with the stage's own function rerun on the @@ -1443,9 +1445,10 @@ def _e8_band_donors( problems["band_donor_layer"] = layer receipt["bands"] = band_rows - # The weights the stage read: every region's incumbents gave up the - # region's donor mass in proportion, so scaling them back to the region's - # whole mass restores them. + # The weights the stage read: every funding stratum's incumbents (a + # region, within its geographic clone where the frame carries clones) + # gave up the stratum's donor mass in proportion, so scaling them back to + # the stratum's whole mass restores them. strata = _funding_strata(household).to_numpy() region_mass = pd.Series(weights).groupby(strata).sum() incumbent_mass = pd.Series(weights[~is_donor]).groupby(strata[~is_donor]).sum() From 07e9747bb68b2eb1674f304ad0e53554107ed4a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:37:48 +0000 Subject: [PATCH 028/108] Follow up the review of the SPI share and band-donor commits (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three reviews of 50b6e629b (stage 16's keyed share) and cd43acf1c (stage 17's per-clone seating) found no must. This takes their should findings and the cheap nits, all inert at K = 1. The ruling items, and the text changes the reviews placed at the next regeneration, are left out and listed below. Band donors (cd43acf1c): - Each band's rounded eligible count is clone 0's, in distinct source households. Pooled over the clones it was a union, which grew with K in every band below the top. Clone 0 walks what one clone walks, so the count is now the same at every K. The per-clone refusal is unchanged. - A band's carrier_mean_age is also withheld where the band seats fewer than three households per clone. At K = 1 a band's carriers rest on at most its seats in source households, so no K = 1 receipt changes. - The result docstring and the receipt comment name the fields that are rounded, and say that the band rows' donor and carrier counts and donor_count stay exact row counts, because the gate reads them. The module docstring says the seats are sized on eligible households counted in source households, and when clone 0's weights and funding factors nest. - Gate: a seating_scale in neither form is named by its type, or as missing, in the failure, the full-scale message and the details, so a replayed old receipt's exact scale does not reach the verdict. Share (50b6e629b): the spine gate kernel hashes stochastic_assignment beside stage_health, since the latent-attribute checks' band moved there. Changelog fragments: stage 16's says what stays bit for bit without the clone column (the strata, given the same copies), that stage 17's funding factors and mass scale move by rounding under the redraw, and that the empty-stratum refusal reads the new draw. Stage 17's drops the clone-0 premise for the exact-total correction and states the nesting condition, the per-band counts, the mean-age rule and the gate message. Corrections to the earlier messages, which stay as they are: - 50b6e629b: under the redraw, stage 17 keeps its seating, carriers, bands and seating scale, but its mass scale and funding factors are not equal. The funding strata's masses sum the redrawn copies' weights, so the factors, and the mass scale where it binds, move in their last bits, and the incumbent and donor weights with them (the reviews' synthetic probes: within about 1e-11 relative). The empty-stratum refusal reads the new draw, so a frame that built before can refuse, and one that refused can build; the reviews found both on synthetic frames. - cd43acf1c: the seating nests exactly at any K, but clone 0's weights and funding factors nest only while the common mass scale is the one K = 1 takes (one at both, or bound by a clone-0 region). The synthetic tests hold it at one. After stage 16 the clones are not exact replicas, and in the reviews' synthetic probes the exact-total correction often landed outside clone 0, at stage 16 too. - Both: rtol 1e-12 is the small synthetic frames' figure. A corrected row misses by a few ulps of the total, so its relative miss grows about as rows x K x 2.2e-16 (design arithmetic). What moves: no value at K = 1. On six synthetic frame and plan configurations at K = 1, the stage-17 evidence and the frame content identity equal cd43acf1c's; at K = 3 and 4 only carrier_mean_age differs, withheld in more bands. Every UK stage kernel hashes graph_evidence, so every spine node re-keys (54 of the spine graph's 79 kernel hashes change, the root's among them; 50b6e629b had re-keyed every node already), as do the dense population kernels and the spine gates. No declaration changes: source_stages.json and the coverage manifest are current, and the H2 fixture is untouched. Tests: - K = 3: the carriers per region are recounted in source households (SOUTH_EAST reads '<3' over three or more rows). With the rounding patched out, each band's eligible count equals clone 0's recount and is below the pooled one; the rounded counts equal K = 1's. - K = 3 and 4: the one-seat bands withhold the mean age, though their carriers come from three or more source households. - The K = 1 per-band refusal message is pinned. - The share gate resolves a sigma-2 rule below sigma 4, and refuses a NaN or infinite sigma. A sample fraction whose 10,000 x f is not whole keeps the fraction in the expected count and the share. - The gate names a float or a missing seating scale by type. The gate kernel's hash includes stochastic_assignment. - Ten mutations on scratch copies of the source each fail the intended test: the six the reviews found missed, a pooled per-band count, and one each for the mean-age rule, the gate message and the gate hash. Left out: the draw text's "(one before uk.full.expand)", which waits for María; clone_count in the stage-16 receipt, with (b); and the declaration texts the reviews put at the next stage-16 and stage-17 text regeneration, so that H2 is not regenerated for text alone (the weight rule's band donor count, the stage-17 notes' seats counted in source households, the stage-16 notes' draw on the id alone). Disclosure-control items from the reviews went to María separately. The touched engine-free files pass, with ruff. source_stages.json and the coverage manifest are current. Co-Authored-By: Claude Opus 5.5 --- .../931-band-donors-per-clone.changed.md | 6 +- .../931-spi-support-keyed-share.changed.md | 6 +- .../build/uk_runtime/graph_evidence.py | 18 +- .../build/uk_runtime/spi_band_donors.py | 48 ++++-- .../build/uk_runtime/stage_health.py | 6 +- .../engine_free/uk/test_uk_graph_evidence.py | 4 + .../engine_free/uk/test_uk_spi_band_donors.py | 154 +++++++++++++++++- .../tests/engine_free/uk/test_uk_spi_spine.py | 19 ++- .../engine_free/uk/test_uk_spi_support.py | 17 ++ .../engine_free/uk/test_uk_stage_health.py | 21 ++- 10 files changed, 261 insertions(+), 38 deletions(-) diff --git a/changelog.d/931-band-donors-per-clone.changed.md b/changelog.d/931-band-donors-per-clone.changed.md index 592ede124b..5d41126774 100644 --- a/changelog.d/931-band-donors-per-clone.changed.md +++ b/changelog.d/931-band-donors-per-clone.changed.md @@ -1,7 +1,7 @@ The SPI income band donors (`spi_income_band_donors`) are seated within each geographic clone (#931). Where the households carry `household_clone_index`, the stage runs its Efraimidis-Spirakis walk once in each clone, over that clone's candidates, with the same person-keyed uniforms, keys and tie order, and the same seats in every clone (María's rulings of 7 Nov). The seats and the household share's cap read the eligible households counted in distinct `source_household_id` values, so they do not depend on the clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, seats exactly the one-clone donors, carriers and bands. Each donor weighs its band's published taxpayers over the clone count times the band's donor count. A spine without the column, as every spine is today, is one clone. -- **Funding.** A donor's funding stratum is its region within its clone, a derived stratum (the manifest's `funding_strata` stays `[region]`), so every clone's regions conserve their own mass. One mass scale, the smallest any clone's region allows, serves every clone, so each band keeps one donor weight and the stage gate's and the identity tool's band-mass reconciliations hold as before. The exact-total correction still lands on one incumbent row, never on a donor, so a band's donors stay equal bit for bit at any clone count; where the clones are exact replicas the corrected row is clone 0's, so clone 0's weights nest to about 1e-12, not bitwise. The mass record's reason is unchanged, so neither the frame's content identity nor the coverage manifest's restated reason moves. -- **Receipt.** The evidence gains `clone_count`, and `funding_strata` lists `household_clone_index` where the column is present (the funding rows' labels then read `region|clone`). `expected_donor_households`, `expected_donor_count` and the donor counts cover every clone; `donor_weight` and `planned_donor_weight` are one clone's weights. Under the receipts' floor of three (`receipt_counts`), the eligible household counts, in all and per band, are counted in source households and rounded in place of the exact counts; `seating_scale` reads `full` or `capped`, since an exact scale below one gives back the eligible count; a band's `carrier_mean_age` reads `<3` when its carriers come from fewer than three source households; and `carrier_region_counts` and the funding rows' `donor_households` are rounded. -- **Gate.** The `spi_income_band_donor_support` check refuses a receipt without a positive integer `clone_count`. A donor's weight times the clone count must stay within the maximum, a band's donors must split evenly across the clones, and at full seating a band seats its plan and never fewer than the clone count times the minimum. The seating scale is read as `full` or `capped`. At one clone the checks and their messages are as before, apart from the seating scale's form. +- **Funding.** A donor's funding stratum is its region within its clone, a derived stratum (the manifest's `funding_strata` stays `[region]`), so every clone's regions conserve their own mass. One mass scale, the smallest any clone's region allows, serves every clone, so each band keeps one donor weight and the stage gate's and the identity tool's band-mass reconciliations hold as before. The exact-total correction still lands on one incumbent row, never on a donor, so a band's donors stay equal bit for bit at any clone count. The seating nests exactly at any clone count. Clone 0's weights and funding factors nest, its weights as the one-clone weights over the clone count, only while the common scale is the one a one-clone spine takes: one at both, or bound by a region of clone 0. Where another clone's region binds, every clone's donors, clone 0's included, take the lower scale. Even then the corrected row, which can sit in any clone, moves by a few ulps of the total, so the weights nest to rounding, not bitwise. The mass record's reason is unchanged, so neither the frame's content identity nor the coverage manifest's restated reason moves. +- **Receipt.** The evidence gains `clone_count`, and `funding_strata` lists `household_clone_index` where the column is present (the funding rows' labels then read `region|clone`). `expected_donor_households`, `expected_donor_count` and the donor counts cover every clone; `donor_weight` and `planned_donor_weight` are one clone's weights. Under the receipts' floor of three (`receipt_counts`), the eligible household counts, in all and per band, are counted in source households and rounded in place of the exact counts, each band's being clone 0's, which a one-clone spine counts alike; `seating_scale` reads `full` or `capped`, since an exact scale below one gives back the eligible count; a band's `carrier_mean_age` reads `<3` when its carriers come from fewer than three source households or the band seats fewer than three households per clone; and `carrier_region_counts` and the funding rows' `donor_households` are rounded. The band rows' donor and carrier counts stay exact row counts, which the gate reads. +- **Gate.** The `spi_income_band_donor_support` check refuses a receipt without a positive integer `clone_count`. A donor's weight times the clone count must stay within the maximum, a band's donors must split evenly across the clones, and at full seating a band seats its plan and never fewer than the clone count times the minimum. The seating scale is read as `full` or `capped`, and a value in neither form is named by its type alone. At one clone the checks and their messages are as before, apart from the seating scale's form. - **Refusals.** A clone short of candidates for a band is refused with the number of such clones and the fewest candidates found. A non-integer, missing or negative clone index, or an index from 0 to K - 1 that no household carries, is refused by count. - **What moves.** At K = 1, no value: on synthetic frames the seating, the carriers, every weight, the funding factors, the mass and seating scales, the mass record and the frame's content identity after the stage are bit for bit the parent commit's, with or without a one-clone index. The receipt takes the forms above. The stage's draw, weight-rule, funding, seating-scale and mass-scale texts and its notes change in `sources.yaml`, `source_stages.json` and the stage dictionary, so the stage's contract hash, the H2 fixture's declarations and the coverage manifest's 25 source digests are regenerated. `spi_income_band_donors` and every node after it re-key, as do the spine gates and the stages that bind `stage_health`. diff --git a/changelog.d/931-spi-support-keyed-share.changed.md b/changelog.d/931-spi-support-keyed-share.changed.md index 6a9f297bef..c8d8b4f8c9 100644 --- a/changelog.d/931-spi-support-keyed-share.changed.md +++ b/changelog.d/931-spi-support-keyed-share.changed.md @@ -1,7 +1,7 @@ The SPI support channel (`spi_support_channel`) copies its zero-weight SPI households as a keyed Bernoulli share instead of a fixed count drawn by row position (#931). A household is copied when its uniform, keyed on its own `household_id` under the stage's seed (42) and the new salt `spi_support_channel:inclusion`, falls below π = min(1, 10,000 × the sample fraction / the distinct `source_household_id` values entering the stage); at π = 1 every household is copied without reading a uniform. The manifest's 10,000 is now the share's numerator, so a one-clone spine's copy count is binomial, with mean min(10,000 × the sample fraction, the source households), instead of exact: about 1,000 at f010, by design arithmetic (10,000 × 0.1). Each household's draw reads only its own id, the seed, the salt and π, so row order does not matter. π counts true sources, so it is the same at every geographic clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, draws exactly the one-clone copies, while the other clones draw on their own ids (María's rulings of 7 Nov). - **Gate.** `e7_spi_synthetic_preclone` is renamed `e7_spi_synthetic_share`. Its declaration carries `maximum_zero_weight_rows_rule: {rule: binomial_share_band, sigma: 4}` instead of the integer 10,000. After the draw the stage resolves it to min(n, ceil(n × (π + band(π, n)))) over the n household rows entering the stage, and runs the unchanged zero-weight gate. The band is the repo's binomial band, moved unchanged from `stage_health.latent_attribute_tolerance` into `stochastic_assignment.binomial_share_tolerance(*, target, rows, sigma)`; `latent_attribute_tolerance` stays, as the same band at four sigma. In `sources.schema.json`, `stack_zero_weight_donors` requires `salt`, a zero-weight declaration takes either the integer maximum or the rule, and `allocate_zero_weight_prior_mass` accepts `geographic_clone_stratum`. The stage's source sentence, draw text and notes describe the share. -- **Per-clone strata.** Where the household table carries `household_clone_index`, the prior allocation's strata also split by geographic clone, so each clone keeps its own mass and its SPI weights depend on its own households and copies only. The manifest declares this as `geographic_clone_stratum: household_clone_index`. A positive-mass stratum without an SPI copy is refused with a count of strata, not their names. Without the column, as on every national spine today, the allocation is unchanged bit for bit. The small-clone-stratum fallback comes later in #931. +- **Per-clone strata.** Where the household table carries `household_clone_index`, the prior allocation's strata also split by geographic clone, so each clone keeps its own mass and its SPI weights depend on its own households and copies only. The manifest declares this as `geographic_clone_stratum: household_clone_index`. A positive-mass stratum without an SPI copy is refused with a count of strata, not their names. Without the column the strata are as before: a frame without it, or with a one-clone index, allocates bit for bit as the region and pension-age strata did, given the same copies. The small-clone-stratum fallback comes later in #931. - **Receipt.** The stage evidence gains `expected_spi_households` (10,000 × the sample fraction), `inclusion_probability` floored to two decimals, `source_households` rounded by the receipts' rule, and the gate's name, rule and sigma, not the maximum it resolved to. `spi_households` stays exact, because the stage gate reads it. -- **Bindings.** spi_spine, the module of stages 14, 16 and 18, rounds the receipt, so `frs_hmrc_spine_leaves` joins the counting stages. spi_support now draws through the keyed-draw helpers, so `high_value_home_support` and `cgt_residential_split`, which bind it, bind them too. -- **What moves.** At K = 1 the SPI copy set is redrawn, with a binomial count around 10,000 × the sample fraction where that is below the source households. Per-stratum SPI weights move with it, and so does every SPI-channel value from `hmrc_spi_income_spine` on, the high-value and CGT selections that rank SPI rows, and calibration. Base FRS rows keep their stage-16 weights, which are their incoming weights times stratum shares that do not depend on the draw, except the row the exact-total correction lands on. On synthetic frames, `spi_income_band_donors` seats the same households, carriers and bands under the old and new draws. The H2 fixture's 135 households at a sample fraction of 135 / 10,000 give π = 1 (10,000 × 0.0135 is 135.0 in float), so its copies are unchanged; its declarations and the stage's contract hash are regenerated, with the coverage manifest's source digests and `source_stages.json`. Every node re-keys. +- **Bindings.** spi_spine, the module of stages 14, 16 and 18, rounds the receipt, so `frs_hmrc_spine_leaves` joins the counting stages. spi_support now draws through the keyed-draw helpers, so `high_value_home_support` and `cgt_residential_split`, which bind it, bind them too. The spine gate kernel's hash names `stochastic_assignment` beside `stage_health`, since the latent-attribute checks' band now lives there. +- **What moves.** At K = 1 the SPI copy set is redrawn, with a binomial count around 10,000 × the sample fraction where that is below the source households. Per-stratum SPI weights move with it, and so does every SPI-channel value from `hmrc_spi_income_spine` on, the high-value and CGT selections that rank SPI rows, and calibration. Base FRS rows keep their stage-16 weights, which are their incoming weights times stratum shares that do not depend on the draw, except the row the exact-total correction lands on. On synthetic frames, `spi_income_band_donors` seats the same households, carriers and bands under the old and new draws, at the same seating scale; its funding factors and mass scale, and with them its donors' and base rows' weights, move only by rounding, since its stratum masses sum the redrawn copies' weights. The empty-stratum refusal reads the new draw, so a small frame that built before can refuse, and one that refused can build. The H2 fixture's 135 households at a sample fraction of 135 / 10,000 give π = 1 (10,000 × 0.0135 is 135.0 in float), so its copies are unchanged; its declarations and the stage's contract hash are regenerated, with the coverage manifest's source digests and `source_stages.json`. Every node re-keys. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py index 8c0b3b00dd..be42907950 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -26,7 +26,7 @@ source_hash, ) -from .. import gate_battery +from .. import gate_battery, stochastic_assignment from ..country_spec import CountrySpec, GatesManifest from ..gate_battery import ( BlockingMode, @@ -378,15 +378,21 @@ def implementation_hash(self) -> str: # on the solve fingerprint and a sign-off between two release builds # leaves an in-graph spine a cache hit (microcosm#1301). The stage # checks live in stage_health, so it keys the gate too (the independent - # review of microcosm#1420, finding 5). Only the transferred phase reads - # the energy level, through energy_pricing, so that phase alone carries - # its hash, as a node param: an energy change re-keys it, not the - # assembled gate every later spine stage waits on (the re-review, E). + # review of microcosm#1420, finding 5), with stochastic_assignment, + # where the binomial band of its latent-attribute checks now lives + # (microcosm#931). Only the transferred phase reads the energy level, + # through energy_pricing, so that phase alone carries its hash, as a + # node param: an energy change re-keys it, not the assembled gate + # every later spine stage waits on (the re-review, E). return hashlib.sha256( canonical_json( { "code": source_hash( - type(self), gate_battery, battery_bindings, stage_health + type(self), + gate_battery, + battery_bindings, + stage_health, + stochastic_assignment, ), "country_resources": uk_solve_fingerprint(), } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py index 807208fc50..69cf857909 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py @@ -26,11 +26,16 @@ On a clone-first spine every household carries its geographic clone (``household_clone_index``, microcosm#931). Each clone then seats its own donors by the same walk over its own candidates, with the same seats per band, -counted in distinct source households, so clone 0 seats exactly what a -one-clone spine seats. Every donor weighs its band's weight over the clone -count, and each clone's regions fund their own donors at one mass scale common -to every clone. A spine without the column is one clone, as every spine is -today: ``uk.full.expand`` writes the index after the spine. +sized on the eligible households counted in distinct source households, so +clone 0 seats exactly what a one-clone spine seats, at any clone count. Every +donor weighs its band's weight over the clone count, and each clone's regions +fund their own donors at one mass scale common to every clone, the smallest +any clone's region allows. Clone 0's weights and funding factors are a +one-clone spine's (its weights over the clone count) only while that scale is +the one a one-clone spine takes: one at both, or bound by a region of clone 0. +Where another clone's region binds, every clone's donors, clone 0's included, +take the lower scale. A spine without the column is one clone, as every spine +is today: ``uk.full.expand`` writes the index after the spine. """ from __future__ import annotations @@ -213,9 +218,15 @@ class UKSPIIncomeBandDonorResult: """Frame with the reserved band rows and the receipt the gate reads. ``seating_scale`` is the exact scale the seats were cut at; the receipt - records it as full or capped. ``eligible_households`` and the counts in - the band, funding and carrier rows are already in their receipt forms: - distinct source households, rounded (UKDS disclosure control). + records it as full or capped. These counts are already in their receipt + forms, counted in distinct source households and rounded (UKDS disclosure + control): ``eligible_households``, each band row's ``eligible_households`` + (clone 0's, which a one-clone spine counts alike), the funding rows' + ``donor_households`` and ``carrier_region_counts``. A band row's + ``carrier_mean_age`` is withheld below three carrier source households or + three seats per clone. The band rows' ``donor_households``, ``carriers`` + and ``carrier_regions`` and ``donor_count`` stay exact row counts, which + the stage gate reads exactly; ``expected_donor_count`` is the plan's. """ frame: Frame @@ -694,7 +705,14 @@ def stack_spi_income_band_donors( f"each of the {clone_count} geographic clones; {len(short)} " f"clone(s) found fewer, the fewest {min(short)}." ) - band_eligible[band] = distinct_count(candidate_sources, eligible) + # The receipt's count for the band is clone 0's, in distinct source + # households: clone 0 walks exactly what one clone walks, so the count + # does not depend on the clone count. The refusal above reads every + # clone's. + first_clone = clone_positions[0] + band_eligible[band] = distinct_count( + candidate_sources[first_clone], eligible[first_clone] + ) uniforms = stable_identity_uniforms( candidate_ids, seed=seed, @@ -855,10 +873,13 @@ def stack_spi_income_band_donors( ) validate_uk_national_frame(result) - # The receipt counts donors and carriers in distinct source households - # and rounds them, withholding a mean resting on fewer than three (UKDS - # disclosure control, receipt_counts): a clone's copy of a household is - # the same survey household. + # The receipt rounds its counts of survey households, counted in distinct + # source households, since a clone's copy of a household is the same + # survey household (UKDS disclosure control, receipt_counts): the eligible + # counts, the funding rows' donors and the carriers per region. A band's + # mean age is withheld below three carrier source households or three + # seats per clone. The band rows' donor and carrier counts stay exact row + # counts, because the stage gate reads them exactly. donor_sources = source_household_ids( donor_household, stage=UK_SPI_INCOME_BAND_DONORS_STAGE_NAME ) @@ -895,6 +916,7 @@ def stack_spi_income_band_donors( "carrier_mean_age": ( float(carrier_ages[carriers].mean()) if carrier_support >= UK_DISCLOSURE_FLOOR + and seats[band] >= UK_DISCLOSURE_FLOOR else UK_WITHHELD ), "carrier_regions": int(carrier_region[carriers].nunique()), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index cd177c7372..2fbbd61472 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -2060,11 +2060,13 @@ def _spi_income_band_donor_support_gate( else f" ({minimum_donors} in each of the {clone_count} geographic clones)" ) # The receipt records the seating scale as full or capped: an exact scale - # below one gives back the eligible household count. + # below one gives back the eligible household count, so the gate names + # any other value by its type alone. seating = evidence.get("seating_scale") if seating not in _BAND_DONOR_SEATINGS: + seating = "missing" if seating is None else f"a {type(seating).__name__}" failures.append( - f"{stage}: seating_scale {seating!r} is neither 'full' nor 'capped'." + f"{stage}: seating_scale is {seating}, neither 'full' nor 'capped'." ) mass_scale = _finite_number(evidence.get("mass_scale"), label=f"{stage}.mass_scale") if not 0.0 < mass_scale <= 1.0: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py index d32f639d07..659f4705ac 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py @@ -561,6 +561,7 @@ def test_the_spine_gate_kernel_keys_on_its_checks_not_the_energy_level(monkeypat # The re-review of microcosm#1420, E: one kernel hash serves both phases, # so it covers the modules both read and not energy_pricing, which only # the transferred phase's node param carries. + from microcosm.build import stochastic_assignment from microcosm.build.uk_runtime import ( battery_bindings, energy_pricing, @@ -582,4 +583,7 @@ def recording(*objects): kernel.implementation_hash() (modules,) = seen assert {stage_health, battery_bindings, graph_evidence.gate_battery} <= set(modules) + # The latent-attribute checks' binomial band lives in stochastic_assignment + # (microcosm#931), so the gate keys on it too. + assert stochastic_assignment in modules assert energy_pricing not in modules diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index 3d4e02d6d1..566cde2811 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -10,6 +10,7 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import spi_band_donors from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.receipt_counts import UK_WITHHELD, rounded_count @@ -1384,7 +1385,10 @@ def test_clone_zero_seats_what_one_clone_seats_and_every_clone_takes_the_seats( assert set(seated[clone]) != set(seated[0]) # Clone 0's weights, its donors' included, are the one clone's over the - # clone count, to the exact-total corrections clone 0 absorbs. + # clone count while the common mass scale is the one clone's, as here, + # where it is one at both. The exact-total correction moves one incumbent + # row by a few ulps of the total, in whichever clone it lands, so on these + # small frames to rtol 1e-12, not bit for bit. weights_once = _weights_by_id(one.frame) clone_zero = _clone_of(many).loc[lambda clones: clones.eq(0)].index assert set(clone_zero) == set(weights_once.index) @@ -1699,10 +1703,12 @@ def test_the_receipt_counts_source_households_at_the_disclosure_floor() -> None: assert sum(row["carriers"] < 3 for row in capped["bands"]) == 3 -def test_a_mean_age_resting_on_fewer_than_three_source_households_is_withheld() -> None: - # Given one claimant alone the tape lets carry the top band, copied into - # each of three clones, - person, benunit, household = _raw_support_tables(200) +def _one_top_band_candidate(person, household) -> pd.DataFrame: + """Household 1's claimant aged 70, alone in the one cell the top band opens. + + Changes ``person`` in place and returns the propensity table. + """ + elder = person["person_household_id"].eq(1) & person["is_uc_claimant"] person.loc[elder, "age"] = 70 cell = ( @@ -1711,7 +1717,7 @@ def test_a_mean_age_resting_on_fewer_than_three_source_households_is_withheld() int(age_band_lower_bound(np.asarray([70.0]))[0]), ) propensity = _propensity() - propensity = pd.concat( + return pd.concat( [ propensity.loc[propensity["band"] != 2_000_000], pd.DataFrame( @@ -1730,6 +1736,29 @@ def test_a_mean_age_resting_on_fewer_than_three_source_households_is_withheld() ], ignore_index=True, ) + + +def test_a_band_short_of_candidates_on_one_clone_is_refused_by_count() -> None: + # Given one claimant alone the tape lets carry the top band, on a spine + # without the clone index, and three seats for the band, + person, benunit, household = _raw_support_tables(200) + propensity = _one_top_band_candidate(person, household) + frame = _spi_support_frame(person, benunit, household, expected_spi_households=10) + + # then the stage refuses with one clone's message: the seats and the count. + with pytest.raises(ValueError) as caught: + _stack_small(frame, propensity=propensity) + assert str(caught.value) == ( + "SPI income band donors need 3 candidate households with positive weight " + "and propensity for the band from 2000000; found 1." + ) + + +def test_a_mean_age_resting_on_fewer_than_three_source_households_is_withheld() -> None: + # Given one claimant alone the tape lets carry the top band, copied into + # each of three clones, + person, benunit, household = _raw_support_tables(200) + propensity = _one_top_band_candidate(person, household) frame = _clone_first_support_frame(200, 3, tables=(person, benunit, household)) result = _stack_small(frame, propensity=propensity, minimum_donors=1) @@ -1745,6 +1774,119 @@ def test_a_mean_age_resting_on_fewer_than_three_source_households_is_withheld() assert first["carriers"] == 12 and isinstance(first["carrier_mean_age"], float) +def test_a_band_seating_fewer_than_three_per_clone_withholds_its_mean_age() -> None: + # Given the published plan on 60 households, which seats one household in + # three of the bands, in each of three and of four clones, + propensity = _propensity() + for n_clones in (3, 4): + result = stack_spi_income_band_donors( + _clone_first_support_frame(60, n_clones), + propensity=propensity, + **_PUBLISHED, + ) + out = result.frame.table("household").set_index("household_id") + person = result.frame.table("person") + carriers = out.loc[ + person.loc[person[PERSON_IS_SPI_INCOME_BAND_CARRIER], "person_household_id"] + ] + sources = carriers.groupby(SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN)[ + "source_household_id" + ].nunique() + + # then a band seating fewer than three households in each clone + # withholds its carriers' mean age, even where they come from three + # source households or more; the band seating six keeps its mean. + one_seat = [ + row for row in result.band_rows if row["donor_households"] < 3 * n_clones + ] + assert [row["donor_households"] for row in one_seat] == [n_clones] * 3 + assert all(row["carrier_mean_age"] == UK_WITHHELD for row in one_seat) + assert any(sources[row["lower_bound"]] >= 3 for row in one_seat) + (seated,) = [row for row in result.band_rows if row not in one_seat] + assert seated["donor_households"] == 6 * n_clones + assert isinstance(seated["carrier_mean_age"], float) + + +def test_the_receipt_counts_source_households_at_any_clone_count(monkeypatch) -> None: + # Given the published plan on 60 households, as one clone and as three, + propensity = _propensity() + one = stack_spi_income_band_donors( + _support_frame(), propensity=propensity, **_PUBLISHED + ) + many = stack_spi_income_band_donors( + _clone_first_support_frame(60, 3), propensity=propensity, **_PUBLISHED + ) + evidence = many.evidence() + + # then the carriers per region are counted in source households over + # every clone: SOUTH_EAST's carrier rows rest on fewer than three. + out = many.frame.table("household").set_index("household_id") + person = many.frame.table("person") + carriers = out.loc[ + person.loc[person[PERSON_IS_SPI_INCOME_BAND_CARRIER], "person_household_id"] + ] + by_region = carriers.groupby("region")["source_household_id"].nunique() + assert evidence["carrier_region_counts"] == { + str(region): rounded_count(int(count)) for region, count in by_region.items() + } + assert int(carriers["region"].eq("SOUTH_EAST").sum()) >= 3 + assert evidence["carrier_region_counts"]["SOUTH_EAST"] == UK_WITHHELD + # Each band's eligible households are clone 0's, which one clone counts + # alike, so the per-band counts are the same at every clone count. + assert [row["eligible_households"] for row in evidence["bands"]] == [ + row["eligible_households"] for row in one.evidence()["bands"] + ] + + # Before rounding, each band's count is the source households clone 0's + # own walk leaves eligible, recounted here from the cells the tape opens; + # pooled over the clones, the lower bands would count more. + monkeypatch.setattr(spi_band_donors, "rounded_count", int) + exact = stack_spi_income_band_donors( + _clone_first_support_frame(60, 3), propensity=propensity, **_PUBLISHED + ) + raw_person, _, raw_household = _raw_support_tables(60) + claimants = raw_person.loc[raw_person["is_uc_claimant"] & raw_person["age"].ge(16)] + region_of = raw_household.set_index("household_id")["region"] + cells = { + int(row.person_household_id): ( + region_of[row.person_household_id], + str(row.gender), + int(age_band_lower_bound(np.asarray([float(row.age)]))[0]), + ) + for row in claimants.itertuples() + } + seated = _seating(exact) + taken: dict[int, set[int]] = {clone: set() for clone in seated} + rows = {row["lower_bound"]: row for row in exact.band_rows} + pooled_more = False + for band in sorted(BANDS, reverse=True): + positive = propensity.loc[ + (propensity["band"] == band) & (propensity["propensity"] > 0.0) + ] + open_cells = set( + zip( + positive["region"], + positive["gender"], + positive["age_band"], + strict=True, + ) + ) + eligible = { + clone: { + source + for source, cell in cells.items() + if cell in open_cells and source not in taken[clone] + } + for clone in seated + } + assert rows[band]["eligible_households"] == len(eligible[0]) + pooled_more |= len(set().union(*eligible.values())) > len(eligible[0]) + for clone, seats in seated.items(): + taken[clone] |= {source for source, seat in seats.items() if seat == band} + assert pooled_more + assert exact.eligible_households == 60 + + def test_the_stage_gate_reads_a_receipt_by_its_clone_count() -> None: one = _stack_small(_support_frame(200)).evidence() evidence = _stack_small(_clone_first_support_frame(200, 3)).evidence() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py index 11639a6bbf..3440e03f08 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py @@ -1196,6 +1196,20 @@ def test_the_stage_draws_the_declared_count_times_the_sample_fraction() -> None: assert rounded["source_households"] == 400 assert rounded["inclusion_probability"] == 0.25 + # Where 10,000 x f is no whole number, the expected count keeps its + # fraction: it is the share's numerator, never rounded to a copy count. + fraction = 0.012345 + transform = UKSPISupportChannelStageTransform( + stage=_committed_stage("spi_support_channel"), + sample_fraction=fraction, + state_pension_age=66, + ) + transform(_share_frame()) + fractional = transform.checkpoint_metadata()["evidence"] + assert fractional["expected_spi_households"] == 10_000 * fraction + assert not float(fractional["expected_spi_households"]).is_integer() + assert transform.last_result.spi_inclusion_probability == 10_000 * fraction / 400 + def test_an_expected_count_at_the_source_count_copies_every_household() -> None: # As in the H2 fixture: 10,000 x (135 / 10,000) is 135.0 in float, so the @@ -1242,8 +1256,9 @@ def copied(household, clone: int) -> set[int]: ) # Nested weights: K x every clone-0 row is the one-clone row. The - # exact-total correction lands on clone 0's rows, so to rtol 1e-12, not - # bit for bit. + # exact-total correction moves one row by a few ulps of the total, at one + # clone and at K, and not always in clone 0, so on these small frames to + # rtol 1e-12, not bit for bit. one_weights = pd.Series( np.asarray(one.weights_for("household").values), index=one_household["household_id"].to_numpy(), diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py index 76234eabdc..a87846dc1c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py @@ -1173,6 +1173,12 @@ def test_the_share_gate_resolves_to_the_share_plus_its_binomial_band() -> None: ) == n ) + # The band is the declared sigma's: two standard errors resolve lower. + two_sigma = spi_support.spi_share_gate_maximum( + {**_SHARE_RULE, "sigma": 2}, inclusion_probability=share, household_rows=n + ) + assert two_sigma == math.ceil(n * (share + 2.0 * math.sqrt(share * 0.7 / n))) + assert two_sigma < maximum # The unchanged gate passes the resolved maximum and refuses one more. for zero_rows, passed in ((maximum, True), (maximum + 1, False)): synthetic = np.arange(n) < zero_rows @@ -1215,6 +1221,17 @@ def test_the_share_gate_resolves_to_the_share_plus_its_binomial_band() -> None: 0.3, "sigma must be positive", ), + *( + ( + { + **_SHARE_GATE, + "maximum_zero_weight_rows_rule": {**_SHARE_RULE, "sigma": sigma}, + }, + 0.3, + "sigma must be positive", + ) + for sigma in (float("nan"), float("inf")) + ), ( {**_SHARE_GATE, "maximum_zero_weight_rows": 10}, 0.3, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py index 62afee24ba..3855402b05 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py @@ -2355,11 +2355,26 @@ def test_spi_income_band_donor_support_gate_scales_only_off_the_full_sample() -> assert not result.passed and "outside (0, 1]" in " ".join(result.failures) # The receipt records the seating scale as full or capped, never as the # exact scale, which gives back the eligible household count. - result = _spi_income_band_donor_gate({**scaled, "seating_scale": 8 / 1_920}) + # A value in neither form is named by its type alone, so a replayed old + # receipt's exact scale does not reach the gate's verdict. + result = _spi_income_band_donor_gate( + {**scaled, "seating_scale": 8 / 1_920, "sample_fraction": 1.0} + ) assert not result.passed assert ( - "spi_income_band_donors: seating_scale 0.004166666666666667 is neither " - "'full' nor 'capped'." + "spi_income_band_donors: seating_scale is a float, neither 'full' nor 'capped'." + ) in result.failures + assert ( + "spi_income_band_donors: a full-sample build must seat and fund the donors " + "at full scale, got seating_scale a float and mass_scale 0.5." + ) in result.failures + assert result.details["seating_scale"] == "a float" + assert not any("0.0041" in failure for failure in result.failures) + result = _spi_income_band_donor_gate( + {key: value for key, value in scaled.items() if key != "seating_scale"} + ) + assert ( + "spi_income_band_donors: seating_scale is missing, neither 'full' nor 'capped'." ) in result.failures From 14849e6a112cfa62399bac96dd7fe1035c6faacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 03:35:30 +0000 Subject: [PATCH 029/108] Correct the CGT and high-value support totals on whole households first (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stages 42 (cgt_support_split) and 26 (high_value_home_support) share cgt_support._exact_support_weights. It put the exact-total correction on the heaviest unselected household. Where that household could not carry it, the correction went to whichever row spi_support's shared fallback picked, which can be a root or copy of a divided household. As María ruled on 9 Nov: whole households first, capped at the 64 heaviest, then today's shared fallback, with copy_rule naming that one exception. - The helper tries the 64 heaviest households the split left whole (CGT_SUPPORT_CORRECTION_CANDIDATES), heaviest first, tied weights in row order. The first that can carry the correction takes it, so every root and copy keeps bitwise w / n. Only where none can does it call the shared fallback, unchanged. spi_support is not touched. - Whole households at stage 42 are the unselected positive households and the selected ones left at one copy. At stage 26, whose selected households all become six rows, they are the unselected positive ones. - A selected household left at one copy carries the correction only where its corrected weight leaves the selection walk and its one copy as they were (keeps_the_split). Without that check, 11 synthetic frames had a one-copy household lifted to a weight that counts two copies, and the identity tool's recompute reported a family the stage never made. Unselected households sit past their band's crossing, so the check passes them without a rerun. - Both copy_rule texts gain the same clause, built on the cap, in the code dictionaries, sources.yaml and source_stages.json: "..., except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight". Both stages' notes say where the correction lands. The was_wealth and was_lisa notes are untouched. - Docstrings (the split, the helper, the high-value split), the nesting scenario comments in both test files, the K = 13 recount comment and the changelog fragment's Exact total and What moves bullets follow the rule. The fragment also says what stage 26 does. What moves: - At K = 1, nothing where the heaviest unselected household is still the first candidate and carries the correction. It is the first candidate unless a selected household left at one copy weighs as much. Such a household weighs at most 60 / K, so that needs every unselected household to be as light. That household then takes the correction if it keeps the split, which moves a rounding amount between two whole households. - Where the heaviest unselected household cannot carry it, at either stage and any K, a rounding-sized correction moves from the row the shared fallback picked to a whole household. Those rows' weights move by rounding amounts, and later stages can move with them. - Re-keyed: the implementation hashes and stage contract digests of stages 26 and 42 (of the 79 spine kernels, only these two implementation hashes move; high_value_home_support binds cgt_support), and every node keyed on the UK solve fingerprint, since sources.yaml, source_stages.json and the coverage manifest moved. - Regenerated: source_stages.json (the two copy_rule and two notes lines), the coverage manifest (25 source digests) and the H2 fixture (the same four declaration lines in sources/fixture.json, and the two stages' stage_contract_sha256 in uk_spine.json). No H2 value moves: on the fixture's frame, all six calls at stages 26 and 42 find the total already exact. Synthetic evidence, counts only, against a git-archive snapshot of af667b7e8: - Stage 42, 751 runs: the nesting scenario at K = 1, 3, 7, 11, 13 and 15; 40 random binding frames, the four tool_probe classes and a 60-frame sweep at each of those K; and the cap frame. 419 need no correction. In 231 today's first candidate carries it, and the output is the parent's bit for bit. In the 101 where today's rule fell back: - 68 now land on a later whole household within the 64, so the correction moves off a root or copy. That is 63 selected households left at one copy and 5 unselected ones. In 11 of them the check passed over earlier one-copy households. - 33 still fall back, onto a root or copy as before, bit for bit. These include the nesting scenario at K = 7 and 13 and the cap frame. No selected household left at one copy outranked a carrying first candidate. Every total is bitwise on target. - Stage 26, 720 runs (60 random owner frames at the same six K, two stratum settings): 441 need none, 257 land on today's first candidate (the parent's output), and 22 move from a root or copy to a later unselected household. None falls back. - The cap frame: 64 heavy whole households that cannot carry the correction, then a 65th that can. It falls back, and with the cap at 65 the 65th takes it. - Worst case, timed on a synthetic vector of 15 x 78,188 rows (a K = 15 spine): sorting the whole households, 64 candidates that cannot carry it and the fallback take 0.43 s, against 0.02 s for today's one candidate. A one-copy household that could carry it costs one rerun of the selection walk (0.41 s at that size). - Identity tool: on the 68 stage-42 frames where the correction moved, E8's support-split block and clone pairs are green, as they are under today's rule; without the check, 11 were red on support_split_selection_stored. On 23 random owner chains (K = 1, 3 and 7) where stage 26's correction moved, its high-value lineage and fold, the scoped weights and E5 are green. Today's rule failed the fold's 1e-12 tolerance on 2 of them. Tests (engine-free, synthetic): today's first candidate keeps the weights bit for bit as the rule before gave them; a later whole household carries the correction where the heaviest cannot, with every divided row bitwise w / n; a selected household left at one copy can carry it; one that the correction would divide is passed over, and the identity receipt stays green (red without the check); the 65th whole household is past the cap; the nesting scenario falls back only at K = 7 and 13 and stays exact; stage 26's analogue; and the copy_rule pins in the code dictionaries, sources.yaml and source_stages.json. The 66 touched engine-free files pass (2,925 tests), with ruff, and ci_test_plan verify is ok. source_stages.json equals the typed projection, the coverage manifest is current, and through the engine slot the H2 fixture regenerates byte-identically and test_h2_uk_spine_parity passes. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 6 +- .../uk/release_input_coverage_manifest.json | 50 +-- .../src/microcosm/build/uk/source_stages.json | 8 +- .../src/microcosm/build/uk/spec/sources.yaml | 8 +- .../microcosm/build/uk_runtime/cgt_support.py | 115 +++++- .../uk_runtime/high_value_home_support.py | 20 +- .../engine_free/uk/test_uk_cgt_support.py | 380 +++++++++++++++++- .../uk/test_uk_high_value_home_support.py | 136 +++++++ .../uk/test_uk_identity_stability_receipts.py | 49 ++- .../parity/uk_spine/sources/fixture.json | 8 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- 11 files changed, 695 insertions(+), 87 deletions(-) diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index 4c564e4a86..535dc0d2a1 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,6 +1,6 @@ The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. - **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. At every K its count check accepts a family's stored count within the family weights' relative tolerance of 1e-9, because folding the family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where the stage's exact-total correction fell on a copy, and at K > 1 by more than the snap absorbs. So for a family whose weight lies within 1e-9 of a multiple of 60, the check accepts n or n + 1 copies. A stored count moved there on its own is still reported, by the flag and family-weight checks. A family the stage split consistently into one copy more there, every member at w / (n + 1), goes unreported at every K. At K = 1 the exact check used to report it. The receipt's tolerance policy says so. -- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies". The stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. -- **Exact total.** The exact-total correction is unchanged. It lands on the heaviest unselected household, so every copy weighs bitwise w / n, except where that household cannot carry it: spi_support's shared fallback then places it, and that row can be a root or a copy. The row is then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. -- **What moves.** No value moves at K = 1. On synthetic frames, the stage's output is unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged. Its tolerance policy now names the count bracket, and at K = 1 it no longer reports a count problem for a family that folds back within the tolerance of its stored count. The stage evidence gains `clone_count`. `cgt_support_split`'s stage contract moves with its texts. Its implementation hash moves, and so does `high_value_home_support`'s, which binds `cgt_support`. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hash are regenerated. +- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception. Both stages' notes say where the correction lands. The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. +- **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy carries it only where its corrected weight leaves the walk's selection and its one copy as they were, so the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the identity tool's recompute would have found a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy. The row is then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate wherever no selected household left at one copy weighs as much, which needs every unselected household to weigh at most 60 / K. The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. +- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them. The same can happen at stage 42 where a selected household left at one copy weighs as much as every unselected one. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged. Its tolerance policy now names the count bracket, and at K = 1 it no longer reports a count problem for a family that folds back within the tolerance of its stored count. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 34a358b664..91e3b27d1c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "a588c0d8f1ff14f217b300710e4e3172a356625438350df273df9e882dca5ac0", + "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 947c391cbe..b60077decf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3908,7 +3908,7 @@ "stratum_lower": 0.95, "copies": 6, "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", - "copy_rule": "every selected household becomes copies rows at weight / copies", + "copy_rule": "every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", "flag_column": "household_is_high_value_support_copy", "copies_column": "high_value_support_copies", @@ -3926,7 +3926,7 @@ "high_value_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. No value is assigned and no draw is made here." + "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made here." }, { "stage": "regional_home_values", @@ -7403,7 +7403,7 @@ ], "selection": "within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "maximum_copy_weight": 60.0, - "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "flag_column": "household_is_cgt_support_copy", "copies_column": "cgt_support_copies", @@ -7421,7 +7421,7 @@ "cgt_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." }, { "stage": "cgt_incidence_clone", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index c17a0b8938..ec2c492ff3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3329,7 +3329,7 @@ stages: stratum_lower: 0.95 copies: 6 stratum_rule: within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected - copy_rule: every selected household becomes copies rows at weight / copies + copy_rule: every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight id_remapping: copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12) flag_column: household_is_high_value_support_copy copies_column: high_value_support_copies @@ -3344,7 +3344,7 @@ stages: - high_value_support_copies - high_value_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split for the dense graph''s post-geography home value map (microcosm#1240). The value node reads every owner clone''s home value off its assigned constituency''s Land Registry sale distribution at the household''s weighted rank among its region''s owners, so each seat''s homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. No value is assigned and no draw is made here.' + notes: 'Spine-only, deterministic, mass-conserving support split for the dense graph''s post-geography home value map (microcosm#1240). The value node reads every owner clone''s home value off its assigned constituency''s Land Registry sale distribution at the household''s weighted rank among its region''s owners, so each seat''s homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made here.' - stage: regional_home_values survey: HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level) source: uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations @@ -6015,7 +6015,7 @@ stages: - non_residential_property_value selection: within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused maximum_copy_weight: 60.0 - copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies + copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight id_remapping: copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused flag_column: household_is_cgt_support_copy copies_column: cgt_support_copies @@ -6030,7 +6030,7 @@ stages: - cgt_support_copies - cgt_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' + notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' - stage: cgt_incidence_clone survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index b7f2ba9eb6..cec8b32178 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -27,7 +27,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from typing import Any @@ -121,6 +121,22 @@ "HMRC Table 3 income band into light copies at equal weight; every " "household's mass and the total household mass are conserved." ) +#: The exact-total correction is tried on at most this many of the households +#: a support split left whole, heaviest first, before spi_support's shared +#: fallback (microcosm#931; :func:`_exact_support_weights`, which the +#: high-value home support split shares). Each candidate that cannot carry the +#: correction costs a few dozen sums of the frame's weights, so the cap bounds +#: the search on a national frame. +CGT_SUPPORT_CORRECTION_CANDIDATES = 64 +#: The one exception to "every copy carries weight / copies" that the +#: ``copy_rule`` of both support splits declares, built on the cap so the +#: declarations restate it (microcosm#931). +CGT_SUPPORT_EXACT_TOTAL_EXCEPTION = ( + f"except where none of the {CGT_SUPPORT_CORRECTION_CANDIDATES} heaviest " + "households left whole can carry the exact-total correction, when the " + "shared fallback can put it on a root or copy, off weight / copies by " + "about one ulp of the total weight" +) _HOUSEHOLD_SUPPORT_CHANNEL_COLUMN = "household_support_channel" #: The geographic clone index the clone-first spine gives every household #: (``rowwise_dataset.ladder_clone_index_column("household")``, the @@ -583,12 +599,17 @@ def split_cgt_support_households( sit below it, and ``n`` is at most the field's bound of 10,000), records ``k`` in ``cgt_support_copy_index`` and carries a full copy of the household's person and benefit-unit rows. The exact-total correction - lands on the heaviest unselected incumbent so every copy weight stays - bitwise ``w / n``, unless that household cannot carry it: then - spi_support's shared fallback places it, and that row can be a root or - copy (:func:`_exact_support_weights`). Refuses a frame the incidence - clone stage (``cgt_incidence_clone``) has already seen or one this stage - has already split. + lands on a household the split left whole (an unselected one, or a + selected one left at one copy): the first of the + :data:`CGT_SUPPORT_CORRECTION_CANDIDATES` heaviest that can carry it, so + every root and copy weighs bitwise ``w / n``. A selected one carries it + only where its corrected weight leaves the walk's selection and its one + copy as they were, so the rule rerun on the output divides the same + households. Only where none of them can does spi_support's shared + fallback place it, and that row can be a root or copy, then off ``w / + n`` by about one ulp of the total (:func:`_exact_support_weights`). + Refuses a frame the incidence clone stage (``cgt_incidence_clone``) has + already seen or one this stage has already split. """ validate_uk_national_frame(frame) @@ -685,10 +706,38 @@ def split_cgt_support_households( final_benunit = pd.concat([benunit, *copy_benunit], ignore_index=True) final_household = pd.concat([household, *copy_household], ignore_index=True) final_weights = np.concatenate([incumbent_weights, *copy_weights]) + + def keeps_the_split(row: int, weight: float) -> bool: + """Whether whole household ``row`` at ``weight`` leaves the split as it was. + + An unselected household sits past its band's crossing, so its weight + moves neither the walk nor its count of one. A selected household + left at one copy can move both, so it carries the correction only + where the walk rerun on its corrected weight selects the same + households and that weight still counts one copy: the rule rerun on + the stage's output then divides the same households the same way, as + the identity tool's recompute does. + """ + + if not selected[row]: + return True + counted = cgt_support_copy_counts( + [weight], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=clone_count + ) + if int(counted[0]) != 1: + return False + moved = weights.copy() + moved[row] = weight + reselected, _ = select_cgt_support_households( + household_ids, moved, income_band, wealth, support_by_band + ) + return bool(np.array_equal(reselected, selected)) + exact = _exact_support_weights( final_weights, target=old_total, - unselected=np.flatnonzero(~selected & (incumbent_weights > 0.0)), + whole=np.flatnonzero((copies == 1) & (incumbent_weights > 0.0)), + keeps_the_split=keeps_the_split, ) receipt = MassChangeRecord( entity="household", @@ -763,24 +812,49 @@ def split_cgt_support_households( def _exact_support_weights( - values: np.ndarray, *, target: float, unselected: np.ndarray + values: np.ndarray, + *, + target: float, + whole: np.ndarray, + keeps_the_split: Callable[[int, float], bool] | None = None, ) -> Weights: """Importance weights summing bitwise to ``target``. - The correction lands on the heaviest unselected incumbent so every split - weight stays exactly ``w / n``. When that correction is not representable, - or no unselected incumbent is positive, it falls back to spi_support's - shared multi-candidate correction, whose candidates (the heaviest, last, - first, lightest and middle positive rows) can be a root or copy of a - split household. + ``whole`` gives the positions of the households a support split left + whole: the positive-weight rows it did not divide, whose weights it left + as they were (the unselected households, and at the CGT split the + selected ones left at one copy). The correction is tried on the + :data:`CGT_SUPPORT_CORRECTION_CANDIDATES` heaviest of them, heaviest + first and tied weights in row order, and lands on the first that can + carry it, so every root and copy of a divided household keeps bitwise + ``w / n`` (microcosm#931). A candidate carries it where the correction + is representable on it and ``keeps_the_split``, where given, accepts its + corrected weight: the CGT split refuses a selected household left at one + copy whose corrected weight would move the walk's selection or count a + second copy. The first candidate is the heaviest unselected household, + the one corrected before, wherever no selected household left at one + copy outweighs it or ties it from an earlier row; a total already exact + comes back unchanged. + + Only where none of them can carry it, or there is none, does + spi_support's shared multi-candidate correction place it, unchanged: its + candidates (the heaviest, last, first, lightest and middle positive rows) + can be a root or copy of a divided household, which is then off ``w / n`` + by about one ulp of the total. On the CGT nesting scenario at 7 and 13 + clones, for one, every whole household sits in the half of NumPy's + pairwise sum whose other half carries exactly half an ulp of the total, + so no change to its weight rounds the total onto the target. """ - if unselected.size: - heaviest = int(unselected[np.argmax(values[unselected])]) + rows = np.asarray(whole, dtype=np.int64) + heaviest_first = rows[np.lexsort((rows, -values[rows]))] + for index in heaviest_first[:CGT_SUPPORT_CORRECTION_CANDIDATES].tolist(): corrected = _exact_total_correction( - values, target=target, correction_index=heaviest + values, target=target, correction_index=index ) - if corrected is not None: + if corrected is not None and ( + keeps_the_split is None or keeps_the_split(index, float(corrected[index])) + ): return Weights(corrected, WeightKind.IMPORTANCE) return _importance_weights_with_exact_total(values, target) @@ -841,7 +915,8 @@ def cgt_support_split_operation_parameters() -> tuple[ "copies = ceil(geographic clone count x household weight / " "maximum_copy_weight), the quotient snapped to the nearest " "integer within clone-count ulps when the clone count " - "exceeds one; every copy carries weight / copies" + "exceeds one; every copy carries weight / copies, " + f"{CGT_SUPPORT_EXACT_TOTAL_EXCEPTION}" ), "id_remapping": ( "copy-id table field cgt_support_split; copy k takes every " diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py index 332ab54b39..7b37aa2b57 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py @@ -33,7 +33,10 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations -from microcosm.build.uk_runtime.cgt_support import _exact_support_weights +from microcosm.build.uk_runtime.cgt_support import ( + CGT_SUPPORT_EXACT_TOTAL_EXCEPTION, + _exact_support_weights, +) from microcosm.build.uk_runtime.household_lineage import ( HIGH_VALUE_SUPPORT_COPY_COLUMN, HIGH_VALUE_SUPPORT_INDEX_COLUMN, @@ -146,7 +149,10 @@ def high_value_support_operation_parameters() -> tuple[ "(cumulative weight before it plus half its own, over the " "region's owner weight) is at or above stratum_lower is selected" ), - "copy_rule": "every selected household becomes copies rows at weight / copies", + "copy_rule": ( + "every selected household becomes copies rows at weight / " + f"copies, {CGT_SUPPORT_EXACT_TOTAL_EXCEPTION}" + ), "id_remapping": ( "copy-id table field high_value_support; copy k takes every " "entity id + k x the field's radix (10**12)" @@ -243,9 +249,11 @@ def split_high_value_support_households( ``high_value_support`` field (so every incoming id must sit below it), records ``k`` in ``high_value_support_copy_index`` and carries a full copy of its person and benefit-unit rows. The exact-total correction lands on - the heaviest unselected incumbent so every copy weight stays bitwise - ``w / copies``, unless that household cannot carry it: then spi_support's - shared fallback places it, and that row can be a root or copy + an unselected household, the first of the 64 heaviest that can carry it + (:data:`microcosm.build.uk_runtime.cgt_support.CGT_SUPPORT_CORRECTION_CANDIDATES`), + so every root and copy weighs bitwise ``w / copies``. Only where none of + them can does spi_support's shared fallback place it, and that row can be + a root or copy, then off ``w / copies`` by about one ulp of the total (:func:`microcosm.build.uk_runtime.cgt_support._exact_support_weights`). Refuses a frame this stage has already split. """ @@ -336,7 +344,7 @@ def split_high_value_support_households( exact = _exact_support_weights( final_weights, target=old_total, - unselected=np.flatnonzero(~selected & (incumbent_weights > 0.0)), + whole=np.flatnonzero(~selected & (incumbent_weights > 0.0)), ) receipt = MassChangeRecord( entity="household", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index b6364104a6..eb32dcd795 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -4,7 +4,8 @@ import json import math -from dataclasses import replace +from dataclasses import dataclass, replace +from pathlib import Path import numpy as np import pandas as pd @@ -12,6 +13,7 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.uk_runtime import cgt_support from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_INVESTABLE_WEALTH_COLUMNS, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, @@ -23,6 +25,8 @@ CGT_SUPPORT_COPIES_COLUMN, CGT_SUPPORT_COPY_BOUND_REFUSAL, CGT_SUPPORT_COPY_INDEX_COLUMN, + CGT_SUPPORT_CORRECTION_CANDIDATES, + CGT_SUPPORT_EXACT_TOTAL_EXCEPTION, CGT_SUPPORT_EXPECTED_MASS, CGT_SUPPORT_HEADROOM, CGT_SUPPORT_INCOME_BAND_LOWER_BOUNDS, @@ -51,6 +55,10 @@ load_hmrc_cgt_joint_distribution, ) from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.spi_support import ( + _exact_total_correction, + _importance_weights_with_exact_total, +) from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate from microcosm.frame import WeightKind from test_support.microcosm_build.uk_cgt_support import ( @@ -1107,15 +1115,104 @@ def test_split_records_pool_exhaustion_and_empty_bands_without_refusing(): assert result.frame.weights_for("household").total == 502.0 -def test_split_falls_back_when_every_incumbent_is_selected(): +# --- where the exact-total correction lands (microcosm#931) --------------------- +# +# The split leaves whole every positive-weight household it does not divide: +# the unselected ones and the selected ones left at one copy. The correction +# is tried on the 64 heaviest of them, heaviest first, and spi_support's +# shared fallback, whose row can be a root or copy, runs only where none of +# them can carry it. + + +@dataclass(frozen=True) +class _CorrectionCall: + """What the split handed the exact-total correction, and its selection.""" + + values: np.ndarray + target: float + whole: np.ndarray + selected: np.ndarray + + +def _recorded_correction( + monkeypatch, frame, top_cells, *, clone_count: int = 1, headroom=None +): + """The split on ``frame``, with what it handed the correction recorded.""" + + calls: list[tuple[np.ndarray, float, np.ndarray]] = [] + selections: list[np.ndarray] = [] + correct = cgt_support._exact_support_weights + select = cgt_support.select_cgt_support_households + + def recording_correct(values, *, target, whole, **keywords): + calls.append((np.array(values), target, np.array(whole))) + return correct(values, target=target, whole=whole, **keywords) + + def recording_select(*args, **kwargs): + selected, rows = select(*args, **kwargs) + selections.append(np.array(selected)) + return selected, rows + + keywords = {} if headroom is None else {"headroom": headroom} + with monkeypatch.context() as patch: + patch.setattr(cgt_support, "_exact_support_weights", recording_correct) + patch.setattr(cgt_support, "select_cgt_support_households", recording_select) + result = split_cgt_support_households( + frame, + distribution=support_distribution(top_cells), + parameters=PARAMETERS, + clone_count=clone_count, + **keywords, + ) + ((values, target, whole),) = calls + # The first selection is the stage's; any later one reruns the walk for + # a selected household left at one copy that could carry the correction. + return result, _CorrectionCall(values, target, whole, selections[0]) + + +def _moved_rows(result, call: _CorrectionCall) -> list[int]: + """The rows whose weight the correction moved, as positions.""" + + weights = np.asarray(result.frame.weights_for("household").values) + assert result.frame.weights_for("household").total == call.target + return np.flatnonzero(weights != call.values).tolist() + + +def _correction_before(call: _CorrectionCall) -> tuple[np.ndarray, bool]: + """The correction before microcosm#931's rule, and whether its one + candidate carried it: the heaviest unselected household (the first of + tied weights), else spi_support's shared fallback.""" + + unselected = call.whole[~call.selected[call.whole]] + if unselected.size: + heaviest = int(unselected[np.argmax(call.values[unselected])]) + corrected = _exact_total_correction( + call.values, target=call.target, correction_index=heaviest + ) + if corrected is not None: + return corrected, True + fallback = _importance_weights_with_exact_total(call.values, call.target) + return np.asarray(fallback.values), False + + +def _carries(call: _CorrectionCall, row: int) -> bool: + return ( + _exact_total_correction(call.values, target=call.target, correction_index=row) + is not None + ) + + +def test_split_falls_back_when_it_leaves_no_household_whole(monkeypatch): + # Both households are divided, so no household is left whole to carry + # the exact-total correction, and spi_support's shared fallback places + # it: on the first root here, off w / n by a rounding amount. frame = support_frame( weights=[130.0, 121.0], wealth=[5.0, 6.0], incomes=[BAND_INCOMES[0]] * 2 ) - result = split_cgt_support_households( + result, call = _recorded_correction( + monkeypatch, frame, - distribution=support_distribution({0: {250_000: 1_000.0}}), - parameters=PARAMETERS, - clone_count=1, + {0: {250_000: 1_000.0}}, headroom=SCENARIO_HEADROOM, ) @@ -1123,6 +1220,76 @@ def test_split_falls_back_when_every_incumbent_is_selected(): assert result.copies_created == 4 assert result.frame.weights_for("household").total == 251.0 assert result.band_rows[0]["pool_exhausted"] is True + assert call.whole.size == 0 + assert float(call.values.sum()) != call.target + assert _moved_rows(result, call) == [0] + + +def test_a_selected_household_left_at_one_copy_can_carry_the_correction( + monkeypatch, +): + # All three are selected and the 40 household is left at one copy, the + # one household the split leaves whole, so it carries the correction. The + # rule before tried unselected households only, so the shared fallback + # corrected the first root, off w / n. + frame = support_frame( + weights=[187.9, 91.8, 40.0], + wealth=[5.0, 6.0, 7.0], + incomes=[BAND_INCOMES[0]] * 3, + ) + result, call = _recorded_correction( + monkeypatch, + frame, + {0: {250_000: 1_000.0}}, + headroom=SCENARIO_HEADROOM, + ) + + household = result.frame.table("household") + assert call.selected.tolist() == [True, True, True] + assert household[CGT_SUPPORT_COPIES_COLUMN].tolist()[:3] == [4, 2, 1] + assert call.whole.tolist() == [2] + assert float(call.values.sum()) != call.target + before, carried = _correction_before(call) + assert not carried + assert np.flatnonzero(before != call.values).tolist() == [0] + assert _moved_rows(result, call) == [2] + weights = _household_weights(result.frame) + assert weights.loc[3] == pytest.approx(40.0, rel=1e-12) + for household_id, weight, copies in ((1, 187.9, 4), (2, 91.8, 2)): + family = [household_id + k * COPY_RADIX for k in range(copies)] + assert weights.loc[family].tolist() == [weight / copies] * copies + + +def test_a_household_the_correction_would_divide_is_passed_over(monkeypatch): + # All four are selected. The 60 household is left at one copy, the + # heaviest the split leaves whole, but the correction it could carry + # lifts it above 60, where the rule counts two copies, so the rule rerun + # on the output (as the identity tool reruns it) would divide it. It is + # passed over, and the 31.5 household, also at one copy, carries it. + frame = support_frame( + weights=[400.0, 268.7, 60.0, 31.5], + wealth=[5.0, 6.0, 7.0, 8.0], + incomes=[BAND_INCOMES[0]] * 4, + ) + result, call = _recorded_correction( + monkeypatch, + frame, + {0: {250_000: 1_000.0}}, + headroom=SCENARIO_HEADROOM, + ) + + assert call.selected.all() + assert call.whole.tolist() == [2, 3] + corrected = _exact_total_correction( + call.values, target=call.target, correction_index=2 + ) + assert corrected is not None + assert cgt_support_copy_counts([corrected[2]], clone_count=1).tolist() == [2] + assert _moved_rows(result, call) == [3] + weights = np.asarray(result.frame.weights_for("household").values) + assert weights[2] == 60.0 + assert weights[3] == pytest.approx(31.5, rel=1e-12) + assert cgt_support_copy_counts([weights[3]], clone_count=1).tolist() == [1] def test_split_counts_suppressed_cells_and_reports_channels_when_present(): @@ -1284,12 +1451,16 @@ def _leaves(value): # more than its pool, so the whole pool is selected at every clone count and # the selection nests exactly; band 37,700 asks for nothing, so its household # (9,000) is the first candidate for the exact-total correction (on clone 0, -# the first heaviest). At seven (and 13) clones its clone 0 has no -# representable correction, so the shared fallback puts it on the last copy -# of the 7,620 family's last clone. That copy is then off w / n by about an -# ulp of the frame's total weight (thousands of ulps of its own, a relative -# error below 1e-12), within the weights' nesting tolerance and the gate's; -# at one, three, 11 and 15 clones no correction is needed. +# the first heaviest), and the light one, selected at one copy, is the other +# household the split leaves whole. At seven (and 13) clones none of their 14 +# (26) clones can carry the correction: all sit in the half of NumPy's +# pairwise sum whose other half carries exactly half an ulp of the total, so +# no change to one of their weights rounds the total onto the target. The +# shared fallback then puts it on the last copy of the 7,620 family's last +# clone, the one exception copy_rule names. That copy is then off w / n by +# about an ulp of the frame's total weight (thousands of ulps of its own, a +# relative error below 1e-12), within the weights' nesting tolerance and the +# gate's; at one, three, 11 and 15 clones no correction is needed. NESTING_WEIGHTS = [ 900.0, @@ -1539,6 +1710,170 @@ def test_the_stage_gate_reads_the_receipts_clone_count(): assert frame.weights_for("household").total == pytest.approx(460.0, rel=1e-12) +# --- whole households past the first candidate (microcosm#931) ------------------ +# +# The cap frames put the households the split leaves whole where NumPy's +# pairwise sum decides whether they can carry the correction. Heavy households +# sit in band 37,700, which asks for no support, so the split leaves them +# whole, and so is the light one after them; the last household, in the +# exhausting band 0, is divided into 67 rows (4,004.74 / 60 = 66.7). The +# 132-row vector the split corrects is summed as two halves split at row 64: +# every heavy household sits in the first, the light one, the divided root +# and its copies in the second. The first half sums inside the total's binade +# and the second carries exactly half an ulp of the total, so round half to +# even keeps every change to a heavy household's weight off the target, while +# the light household and the copies can carry it. Each test asserts that +# premise before the rule. + +_CAP_HEAVY = [1_100.0 + index for index in range(64)] +_CAP_LIGHT = 1_000.0 +_CAP_DIVIDED = 4_004.74 +_CAP_COPIES = 67 + + +def _cap_frame(heavy): + """``heavy`` whole households, the light one, then the divided one.""" + + weights = [*heavy, _CAP_LIGHT, _CAP_DIVIDED] + return support_frame( + weights=weights, + wealth=[1.0] * (len(weights) - 1) + [1e6], + incomes=[BAND_INCOMES[37_700]] * (len(weights) - 1) + [BAND_INCOMES[0]], + ) + + +def test_where_the_first_candidate_carries_it_the_weights_are_as_before( + monkeypatch, +): + # The first candidate is the heaviest unselected household, the one the + # rule before corrected, wherever no selected household left at one copy + # outweighs it (such a household weighs at most 60 / K). So wherever it + # can carry the correction, the weights are the ones the rule before + # gave, bit for bit: on the reference scenario (id 5) and on random + # one-clone frames that ask each band for about 30% of its mass. + rng = np.random.default_rng(7) + frames = [(_scenario_frame(), SCENARIO_TOP_CELLS, SCENARIO_HEADROOM)] + for _ in range(12): + weights = rng.lognormal(4.5, 1.0, 120) + wealth = rng.lognormal(11.0, 1.5, 120) + bands = rng.choice([0, 37_700], size=120) + top = { + int(band): { + 250_000: float(np.round(0.3 * weights[bands == band].sum() / 2.5, 1)) + } + for band in (0, 37_700) + } + frame = support_frame( + weights=weights.tolist(), + wealth=wealth.tolist(), + incomes=[BAND_INCOMES[int(band)] for band in bands], + ) + frames.append((frame, top, None)) + carried = 0 + for frame, top, headroom in frames: + result, call = _recorded_correction(monkeypatch, frame, top, headroom=headroom) + before, first_carried = _correction_before(call) + weights = np.asarray(result.frame.weights_for("household").values) + assert result.frame.weights_for("household").total == call.target + if not first_carried or float(call.values.sum()) == call.target: + continue + carried += 1 + unselected = call.whole[~call.selected[call.whole]] + first = call.whole[np.lexsort((call.whole, -call.values[call.whole]))][0] + assert first == unselected[np.argmax(call.values[unselected])] + assert weights.tobytes() == before.tobytes() + assert carried >= 3 + + +def test_the_correction_passes_over_whole_households_that_cannot_carry_it( + monkeypatch, +): + # 63 heavy households and a zero-weight one fill the first half, so the + # light household is the 64th and last whole one the rule tries. The rule + # before tried the heaviest unselected household only, and the shared + # fallback then corrected the last copy; now the light one carries it and + # every row of the divided household stays bitwise w / n. + result, call = _recorded_correction( + monkeypatch, _cap_frame([*_CAP_HEAVY[1:], 0.0]), NESTING_TOP_CELLS + ) + + assert call.whole.tolist() == [*range(63), 64] + assert not call.selected[call.whole].any() + assert float(call.values.sum()) != call.target + assert not any(_carries(call, row) for row in range(63)) + assert _carries(call, 64) + before, carried = _correction_before(call) + assert not carried + assert np.flatnonzero(before != call.values).tolist() == [len(call.values) - 1] + + assert _moved_rows(result, call) == [64] + weights = np.asarray(result.frame.weights_for("household").values) + assert weights[64] == pytest.approx(_CAP_LIGHT, rel=1e-12) + assert weights[65:].tolist() == [_CAP_DIVIDED / _CAP_COPIES] * _CAP_COPIES + + +def test_the_correction_tries_only_the_64_heaviest_whole_households(monkeypatch): + # With all 64 heavy households before it, the light one is the 65th + # whole household, past the cap: none of the 64 can carry the + # correction, so the shared fallback places it, on the last copy, though + # the 65th could have. One candidate more and the 65th carries it. + assert CGT_SUPPORT_CORRECTION_CANDIDATES == 64 + result, call = _recorded_correction( + monkeypatch, _cap_frame(_CAP_HEAVY), NESTING_TOP_CELLS + ) + + assert call.whole.tolist() == list(range(65)) + assert float(call.values.sum()) != call.target + assert not any(_carries(call, row) for row in range(64)) + assert _carries(call, 64) + assert _moved_rows(result, call) == [len(call.values) - 1] + fallback = _importance_weights_with_exact_total(call.values, call.target) + weights = np.asarray(result.frame.weights_for("household").values) + assert weights.tobytes() == np.asarray(fallback.values).tobytes() + assert weights[-1] == pytest.approx(_CAP_DIVIDED / _CAP_COPIES, rel=1e-12) + + monkeypatch.setattr(cgt_support, "CGT_SUPPORT_CORRECTION_CANDIDATES", 65) + wider, wider_call = _recorded_correction( + monkeypatch, _cap_frame(_CAP_HEAVY), NESTING_TOP_CELLS + ) + assert _moved_rows(wider, wider_call) == [64] + wider_weights = np.asarray(wider.frame.weights_for("household").values) + assert wider_weights[65:].tolist() == [_CAP_DIVIDED / _CAP_COPIES] * _CAP_COPIES + + +@pytest.mark.parametrize("clone_count", [1, 3, 7, 11, 13, 15]) +def test_the_nesting_scenario_corrects_a_copy_only_where_no_whole_household_can( + monkeypatch, clone_count: int +): + # The split leaves two households of each clone whole: the 9,000 one and + # the light 40 one, selected at one copy. At 7 and 13 clones none of them + # can carry the correction, so the shared fallback still puts it on the + # last row, the last copy of the 7,620 family's last clone, and the total + # stays bitwise on target. At 1, 3, 11 and 15 none is needed. + frame = clone_first_frame( + support_frame( + weights=NESTING_WEIGHTS, wealth=NESTING_WEALTH, incomes=NESTING_INCOMES + ), + clone_count, + ) + result, call = _recorded_correction( + monkeypatch, frame, NESTING_TOP_CELLS, clone_count=clone_count + ) + + assert call.whole.size == 2 * clone_count + moved = _moved_rows(result, call) + if clone_count not in (7, 13): + assert float(call.values.sum()) == call.target + assert moved == [] + return + assert not any(_carries(call, row) for row in call.whole.tolist()) + assert moved == [len(call.values) - 1] + last = result.frame.table("household").iloc[-1] + assert last["source_household_id"] == 4 + assert last["household_clone_index"] == clone_count - 1 + assert last[CGT_SUPPORT_COPY_INDEX_COLUMN] == 126 + + # --- the closed-world dictionary and the transform ------------------------------ @@ -1580,7 +1915,10 @@ def test_operation_dictionary_restates_the_reviewed_design(): "copies = ceil(geographic clone count x household weight / " "maximum_copy_weight), the quotient snapped to the nearest integer " "within clone-count ulps when the clone count exceeds one; every copy " - "carries weight / copies" + "carries weight / copies, except where none of the 64 heaviest " + "households left whole can carry the exact-total correction, when the " + "shared fallback can put it on a root or copy, off weight / copies by " + "about one ulp of the total weight" ) assert payload["flag_column"] == HOUSEHOLD_IS_CGT_SUPPORT_COPY assert payload["copies_column"] == CGT_SUPPORT_COPIES_COLUMN @@ -1773,3 +2111,19 @@ def test_packaged_manifest_declares_the_stage() -> None: "cgt_incidence_clone" ) assert "cgt_band_donors" not in names + + +def test_the_packaged_declarations_restate_the_copy_rule() -> None: + # The copy rule names its one exception identically in the code + # dictionary, sources.yaml and source_stages.json (microcosm#931). + ((_, payload),) = cgt_support_split_operation_parameters() + stage = load_country_spec("uk").sources.stage_map()[CGT_SUPPORT_SPLIT_STAGE_NAME] + (operation,) = stage.operations + assert operation.parameters["copy_rule"] == payload["copy_rule"] + mirror = Path(cgt_support.__file__).parents[1] / "uk" / "source_stages.json" + packaged = json.loads(mirror.read_text()) + (entry,) = [row for row in packaged["stages"] if row["stage"] == stage.stage] + assert entry["operations"][0]["copy_rule"] == payload["copy_rule"] + assert payload["copy_rule"].endswith( + "; every copy carries weight / copies, " + CGT_SUPPORT_EXACT_TOTAL_EXCEPTION + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py index ec092078c8..848493e6e8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py @@ -229,6 +229,142 @@ def _identity_tool(): return module +#: Owner weights of the correction frame (synthetic), lowest value first. +_CORRECTION_OWNERS = [ + 305.47, + 148.26, + 219.59, + 328.65, + 319.35, + 254.19, + 293.18, + 154.34, + 263.21, + 108.39, + 324.55, + 358.98, + 155.78, + 332.99, +] + + +def _correction_frame(): + """63 heavy renters, a zero-weight one, a light one, then 14 owners. + + Renters are never in the stratum, so the stage leaves them whole; at + stratum 0.3 the ten most valuable owners become six rows each, and the + 129-row vector the stage corrects is summed as two halves split at row + 64. The 63 heavy renters sit in the first half, which sums inside the + total's binade, while the second, holding the light renter, the owners + and the copies, carries exactly half an ulp of the total: round half to + even keeps every change to a heavy renter's weight off the target, and + the light one, the 64th whole household by weight, can carry it. + """ + weights = [*(1_101.0 + i for i in range(63)), 0.0, 1_000.0, *_CORRECTION_OWNERS] + n = len(weights) + owners = len(_CORRECTION_OWNERS) + base = _frame(n) + household = base.table("household").copy() + household["tenure_type"] = pd.array( + ["RENT_PRIVATELY"] * (n - owners) + ["OWNED_OUTRIGHT"] * owners, + dtype="string", + ) + household["main_residence_value"] = [150_000.0] * (n - owners) + [ + 100_000.0 * (i + 1) for i in range(owners) + ] + return uk_national_frame( + person=base.table("person"), + benunit=base.table("benunit"), + household=household, + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=np.asarray(weights), + ) + + +def test_the_correction_passes_over_whole_households_that_cannot_carry_it( + monkeypatch, +) -> None: + # The stage corrects through the CGT split's helper (microcosm#931): the + # 64 heaviest households it leaves whole, its unselected ones, heaviest + # first. The rule before tried the heaviest only, and the shared fallback + # then corrected the last copy; now the light renter carries it and every + # root and copy stays bitwise w / 6. + from microcosm.build.uk_runtime.spi_support import ( + _exact_total_correction, + _importance_weights_with_exact_total, + ) + + calls = [] + correct = hv._exact_support_weights + + def recording(values, *, target, whole, **keywords): + calls.append((np.array(values), target, np.array(whole))) + return correct(values, target=target, whole=whole, **keywords) + + monkeypatch.setattr(hv, "_exact_support_weights", recording) + result = hv.split_high_value_support_households( + _correction_frame(), stratum_lower=0.3, copies=6 + ) + ((values, target, whole),) = calls + + def carries(row: int) -> bool: + corrected = _exact_total_correction(values, target=target, correction_index=row) + return corrected is not None + + assert result.households_selected == 10 and len(values) == 129 + heaviest_first = whole[np.lexsort((whole, -values[whole]))] + assert heaviest_first[:64].tolist() == [*range(62, -1, -1), 64] + assert float(values.sum()) != target + assert not any(carries(row) for row in range(63)) + assert carries(64) + fallback = np.asarray(_importance_weights_with_exact_total(values, target).values) + assert np.flatnonzero(fallback != values).tolist() == [128] + + weights = np.asarray(result.frame.weights_for("household").values) + assert result.frame.weights_for("household").total == target + assert np.flatnonzero(weights != values).tolist() == [64] + assert weights[64] == pytest.approx(1_000.0, rel=1e-12) + household = result.frame.table("household") + copies = household[hv.HIGH_VALUE_SUPPORT_COPIES_COLUMN].to_numpy() + sources = household["household_id"].to_numpy() % 10**12 + for source in np.unique(sources[copies == 6]): + family = weights[sources == source] + weight = _CORRECTION_OWNERS[int(source) - 66] + assert family.tolist() == [weight / 6] * 6 + + +def test_the_copy_rule_names_the_one_exception() -> None: + import json + from pathlib import Path + + from microcosm.build.uk_runtime.cgt_support import ( + CGT_SUPPORT_CORRECTION_CANDIDATES, + CGT_SUPPORT_EXACT_TOTAL_EXCEPTION, + ) + + ((_, payload),) = hv.high_value_support_operation_parameters() + expected = ( + "every selected household becomes copies rows at weight / copies, " + "except where none of the 64 heaviest households left whole can carry " + "the exact-total correction, when the shared fallback can put it on a " + "root or copy, off weight / copies by about one ulp of the total weight" + ) + assert CGT_SUPPORT_CORRECTION_CANDIDATES == 64 + assert payload["copy_rule"] == expected + assert expected.endswith(CGT_SUPPORT_EXACT_TOTAL_EXCEPTION) + stage = load_country_spec("uk").sources.stage_map()[ + hv.HIGH_VALUE_SUPPORT_STAGE_NAME + ] + (operation,) = stage.operations + assert operation.parameters["copy_rule"] == expected + packaged = json.loads( + (Path(hv.__file__).parents[1] / "uk" / "source_stages.json").read_text() + ) + (entry,) = [row for row in packaged["stages"] if row["stage"] == stage.stage] + assert entry["operations"][0]["copy_rule"] == expected + + def test_the_split_refuses_incoming_ids_at_its_copy_id_radix() -> None: frame = _frame() household = frame.table("household").copy() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index fd959594ae..97f82dba68 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1108,11 +1108,12 @@ def test_an_artifact_without_the_split_layer_is_refused(self) -> None: #: 7, 21 copies); band 0's pool is exhausted at every clone count, band #: 37,700 asks for nothing, so its household (9,000) is the first candidate #: for the exact-total correction, and the zero-weight household is outside -#: every pool. At 7 and 13 clones that household's clone 0 has no -#: representable correction, so the shared fallback puts it on the last copy -#: of the 7,620 family's last clone, off w / n by about an ulp of the frame's -#: total weight (a relative error below 1e-12, within the 1e-12 tolerances); -#: at 1, 3, 11 and 15 clones no correction is needed. +#: every pool. At 7 and 13 clones no household the split leaves whole (the +#: 9,000 household's clones and the light one's) can carry the correction, so +#: the shared fallback puts it on the last copy of the 7,620 family's last +#: clone, off w / n by about an ulp of the frame's total weight (a relative +#: error below 1e-12, within the 1e-12 tolerances); at 1, 3, 11 and 15 clones +#: no correction is needed. _NESTING_SCENARIO = { "weights": [ 900.0, @@ -1261,8 +1262,9 @@ def test_exact_multiples_of_60_receipt_green_at_any_clone_count( # to a weight whose K x w / 60 sits more ulps above 127 than the # count's snap absorbs, so an exact recount would count 128 in all 11 # clones at 11 and in 12 of the 13 at 13 (the clone whose last copy - # took the exact-total correction re-sums to 127); the tolerance - # accepts the stage's 127. + # took the exact-total correction, which no household left whole + # could carry there, re-sums to 127); the tolerance accepts the + # stage's 127. one_split, one = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) split, many = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) @@ -1372,6 +1374,39 @@ def test_at_one_clone_a_family_folded_just_above_a_multiple_is_green( assert receipt["copies"] == split.copies_created == 259 assert receipt["copies_recomputed"] == 260 + def test_a_household_the_correction_would_divide_keeps_the_receipt_green( + self, monkeypatch + ) -> None: + # Every household is selected, and the heaviest left at one copy + # weighs 60: the correction it could carry would lift it to a weight + # that counts two copies (test_uk_cgt_support.py). The split passes + # it over (microcosm#931), so the recompute divides the households + # the stage divided. Had the 60 household carried the correction, the + # recompute would find a family the stage never made. + from microcosm.build.uk_runtime import cgt_support + + scenario = { + "weights": [400.0, 268.7, 60.0, 31.5], + "wealth": [5.0, 6.0, 7.0, 8.0], + "incomes": [BAND_INCOMES[0]] * 4, + "top_cells": {0: {250_000: 1_000.0}}, + } + _, frame = _clone_first_split_then_cloned(1, scenario) + _, problems = _exhausting_receipt(frame, scenario) + assert problems == {} + + correct = cgt_support._exact_support_weights + + def without_the_check(values, *, target, whole, keeps_the_split=None): + return correct(values, target=target, whole=whole) + + monkeypatch.setattr(cgt_support, "_exact_support_weights", without_the_check) + _, unchecked = _clone_first_split_then_cloned(1, scenario) + _, problems = _exhausting_receipt(unchecked, scenario) + assert problems == { + "support_split_selection_stored": {"missing": 0, "extra": 1} + } + # --- copy ids at the copy-id table's declared radices (microcosm#931) ---------- # diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 71b19df8b2..1ba528d30c 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -563,7 +563,7 @@ ], "grain": "household", "nonnegative_outputs": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", "operations": [ { "adult_minimum_age": 16, @@ -573,7 +573,7 @@ "conservation": "exact_total", "copies_column": "cgt_support_copies", "copy_index_column": "cgt_support_copy_index", - "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", "declared_factor": 1, "expected_support_mass": 145000, "flag_column": "household_is_cgt_support_copy", @@ -3080,14 +3080,14 @@ "artifacts": [], "grain": "household", "nonnegative_outputs": [], - "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. No value is assigned and no draw is made here.", + "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made here.", "operations": [ { "conservation": "exact_total", "copies": 6, "copies_column": "high_value_support_copies", "copy_index_column": "high_value_support_copy_index", - "copy_rule": "every selected household becomes copies rows at weight / copies", + "copy_rule": "every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", "declared_factor": 1, "flag_column": "household_is_high_value_support_copy", "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 62f4640246..f03d9336c5 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"06d6b6f188c766bbc578b91debb5291b1b379679bcf4ff59a39d8b104200ac2c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"fec5bddc81b70da31dcf9a301405049b71b11113bcdd2c2c5950efc8e3a4dc18","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"ed46fd4d2dc5df11ea9fdd0e955d8a7e13c20be5bae275b7a0a2f209a653fc78","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"71433baf763eb12157952507a3b605fd84b45045c75315191d12044d1b97cf4a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} From c3599551b8a487e798cfe38bf4484505a2309619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:41:38 +0000 Subject: [PATCH 030/108] Hold the K = 1 support-copy count exactly where a family's members carry one weight (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since e279b69b0 the identity tool's support-split count check brackets the stored count between the counts at the folded family weight times 1 - 1e-9 and 1 + 1e-9 at every K. Near a multiple of 60 it accepts n or n + 1, so a family the stage split consistently into one copy more there went unreported. As María ruled on 9 Nov, the K = 1 check is now exact wherever a family's members carry one weight, and the bracket stays for the rest. Tool only. - At K = 1, a family the recompute and the stored layer agree on is held exactly where all n members are present, n is above three and every member's clone pair sum is one value c. The stored n must be the count of some w with fl(w / n) == c. Those w are one run of floats, found by stepping with np.nextafter, and the count rises with w, so the run's ends decide. - Why it is sound. The split's own exact-total correction, the incidence clone's and the residential split's each move one row. So a family of more than three members has a member no correction moved. Where at least four members show their two clone halves at equal weights, one of those is uncorrected, and its pair sum is exactly w / n. The clone halves a weight exactly. The #970 anchor scales a clone half by a factor of at most one and adds the rest to the original. The residential split divides only liable clone halves, which the anchor leaves alone. Elsewhere the anchor's and the arms' roundings can move every member of a family to one value a float off w / n. There the hold reaches one float either side of c. - The bracket stays for families of three or fewer, which may have no uncorrected member, for families whose members differ, and for every family at K > 1. - Texts. The _e8_support_split docstring, the count-check comment, the receipt's tolerance_policy, the receipts test's class docstring and the changelog fragment say which families are held exactly. They also say that where the bracket applies it opens "for a family whose source-scaled weight K x w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight)", nit (a) of the 8 Nov re-review, and that copies_recomputed stays the exact recount from the re-summed fold. What moves: tool verdicts only, and the tolerance_policy text; the receipt's keys are unchanged. - At K = 1 the count check reports a family of more than three members that the stage split consistently into one copy more on a multiple of 60, where its members fold to one weight. With the 8 Nov review's floor(q) + 1 stage on the nesting scenario through the clone, it reports all four exact multiples (support_split_copies_stored = 4). The parent's tool reported nothing. - At K = 3 nothing is reported there, as before. - No stage, value, declaration or node key moves, and nothing is regenerated: no node or fingerprint hashes tools/. Synthetic evidence, counts only, against a git-archive snapshot of the parent's tool: - The pair arithmetic, replayed with the stages' own operations for 10 million members per case with n of 4 or more. Untouched pairs sum to exactly w / n. Trimmed pairs, liable pairs with their arm, and both together are at most one float off. Equal halves are never off. - No false support_split_copies_stored on frames the real stages build: - the 8 Nov tool_probe classes and the nesting scenario, split then cloned, at K = 1, 3, 7, 11, 13 and 15, as built and tampered. Their verdicts are the parent's at every K; at K = 1, 986 of 1,383 families are held exactly; - 60 random K = 1 frames, half the weights multiples of 60 and a third of those one to three ulps above, split then cloned. 4,765 of 5,660 families are held exactly; - the nesting scenario and those frames through a hand redraw, the anchor and the residential split, in four mixes of gain groups. 40 to 58 percent of families are held, 161 to 920 of them to within one float. With the reach forced to zero, one correctly counted family there would be reported; - the H2 oracle frame (K = 1), which holds no support family: green under both tools. - Detection through that full chain: the floor(q) + 1 stage gives 524 to 933 count flags across the four mixes. The parent gives none. - The blind spot, by design arithmetic over every n from 4 to 9,998. Held exactly, at most the multiple itself (for 31 of the 9,995 counts) for one copy more, and never for one fewer. Within one float, the multiple and the float either side. The bracket's relative 1e-9 is 4.5 to 9.0 million floats of w on each side. Tests (engine-free, synthetic): - floor(q) + 1 is reported at K = 1, root and copies together, and still unreported at K = 3; - families one to three ulps above a multiple stay green, held exactly; - a corrected member drops its family to the bracket; - a family of three or fewer is left to the bracket; - members all one float off are held to within one float and stay green, where a reach of zero would report them; - through the anchor and the residential split the correct stage is green and three of four floor(q) + 1 families are reported. Two test comments are updated. The eight engine-free files that load the tool or the split pass (338 tests) and ruff is clean. source_stages.json, the coverage manifest and ci_test_plan verify are current. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 4 +- .../uk/test_uk_identity_stability_receipts.py | 373 ++++++++++++++++-- tools/verify_uk_identity_stability.py | 274 +++++++++++-- 3 files changed, 583 insertions(+), 68 deletions(-) diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index 535dc0d2a1..326ceef804 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,6 +1,6 @@ The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. -- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. At every K its count check accepts a family's stored count within the family weights' relative tolerance of 1e-9, because folding the family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where the stage's exact-total correction fell on a copy, and at K > 1 by more than the snap absorbs. So for a family whose weight lies within 1e-9 of a multiple of 60, the check accepts n or n + 1 copies. A stored count moved there on its own is still reported, by the flag and family-weight checks. A family the stage split consistently into one copy more there, every member at w / (n + 1), goes unreported at every K. At K = 1 the exact check used to report it. The receipt's tolerance policy says so. +- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where an exact-total correction fell on a member, and at K > 1 by more than the snap absorbs. So the count check does not recount the fold. At K = 1 a family of more than three members whose clone pair sums are all one weight c is held exactly: its stored count n must be the count of a weight w with fl(w / n) = c. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so such a family has a member no correction moved, whose pair sum only the later stages' roundings can have moved off w / n. Where at least four members show their two clone halves at equal weights, one of those is uncorrected, and equal halves mean the #970 anchor and the residential arms left its pair sum at exactly w / n, so c is exactly w / n. Elsewhere the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member. Every other family, and every family at K > 1, keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. That bracket opens for a family whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a family the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to at most the multiple itself, or the multiple and the float either side of it where the hold reaches one float. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. - **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception. Both stages' notes say where the correction lands. The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. - **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy carries it only where its corrected weight leaves the walk's selection and its one copy as they were, so the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the identity tool's recompute would have found a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy. The row is then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate wherever no selected household left at one copy weighs as much, which needs every unselected household to weigh at most 60 / K. The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. -- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them. The same can happen at stage 42 where a selected household left at one copy weighs as much as every unselected one. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged. Its tolerance policy now names the count bracket, and at K = 1 it no longer reports a count problem for a family that folds back within the tolerance of its stored count. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. +- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them. The same can happen at stage 42 where a selected household left at one copy weighs as much as every unselected one. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the K = 1 hold and the count bracket. Against the exact recount it replaces, at K = 1 it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, but no longer one of three or fewer members, or one whose members fold to different weights. At K > 1 no such family is reported. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 97f82dba68..a997b0c6d9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1169,15 +1169,160 @@ def _exhausting_receipt( return receipt, problems +def _ulps_above(value: float, ulps: int) -> float: + for _ in range(ulps): + value = float(np.nextafter(value, np.inf)) + return value + + +def _floor_plus_one(weights, maximum_copy_weight, *, clone_count): + """A stage bug: ``floor(q) + 1`` copies for ``q = K x w / 60``, not ``ceil(q)``. + + It differs from the stage only where ``q`` is an integer, where it splits + the family consistently into one copy more, root and copies together, + every member at ``w / (n + 1)`` and indexed in full. + """ + + quotient = ( + clone_count * np.asarray(weights, dtype=float) / float(maximum_copy_weight) + ) + return np.maximum(np.floor(quotient).astype("int64") + 1, 1) + + +def _split_with_floor_plus_one(monkeypatch, clone_count: int, scenario): + """``_clone_first_split_then_cloned`` with the split counting ``floor(q) + 1``.""" + + from microcosm.build.uk_runtime import cgt_support + + with monkeypatch.context() as patched: + patched.setattr(cgt_support, "cgt_support_copy_counts", _floor_plus_one) + return _clone_first_split_then_cloned(clone_count, scenario) + + +def _held_families(frame, scenario) -> tuple[dict, dict, int, int]: + """The support-split receipt and problems, with the K = 1 hold's reach. + + Also returns how many families the hold took exactly and how many to + within one float; at ``K > 1``, or where no family is held, both are 0. + """ + + tool = _load_tool() + dividing_to = tool._floats_dividing_to + reaches: list[np.ndarray] = [] + + def recorded(quotients, divisors, floats): + reaches.append(np.asarray(floats)) + return dividing_to(quotients, divisors, floats) + + tool._floats_dividing_to = recorded + problems: dict[str, object] = {} + receipt = tool._e8_support_split( + frame, + problems, + distribution=support_distribution(scenario["top_cells"]), + parameters=PARAMETERS, + permutation_seed=7, + ) + reach = np.concatenate(reaches) if reaches else np.zeros(0, dtype=np.int64) + return receipt, problems, int((reach == 0).sum()), int((reach > 0).sum()) + + +def _through_the_residential_split(twinned): + """A split-then-cloned frame through stages 44 to 46, arms folded away. + + A hand redraw stands in for stage 44: each clone carrier the clone stage + gave a prior takes, by its person id, a liable gain, a gain below the + annual exempt amount, a loss or nothing, so the #970 anchor (45) finds + all four of its groups. The residential split (46) runs on + Table 8 totals sized to the frame's liable mass, as in + ``_residentially_split``, and its arms are then folded onto their + sources, the frame the E8 entry receipts the support split on. + """ + + import dataclasses + + from microcosm.build.uk_runtime.cgt_asset_type import ( + CGT_STOCK_HOUSEHOLD_COLUMNS, + CGT_STOCK_PERSON_COLUMNS, + load_hmrc_cgt_asset_type_facts, + ) + from microcosm.build.uk_runtime.cgt_residential_split import ( + HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, + split_cgt_residential_households, + ) + from microcosm.build.uk_runtime.cgt_structure import anchor_cgt_incidence + from microcosm.build.uk_runtime.national_frame import uk_household_weight_kind + + person = twinned.table("person").copy() + household = twinned.table("household").copy() + for column in CGT_STOCK_PERSON_COLUMNS: + person[column] = person.get(column, 0.0) + for column in CGT_STOCK_HOUSEHOLD_COLUMNS: + household[column] = household.get(column, 0.0) + ids = person["person_id"].to_numpy() + group = ids % 4 + gains = np.select( + [group == 0, group == 1, group == 2], + [20_000.0 + 10.0 * (ids % 7), 500.0 + (ids % 11), -300.0 - (ids % 5)], + 0.0, + ) + carrier = person["capital_gains"].to_numpy() != 0.0 + person["capital_gains"] = np.where(carrier, gains, 0.0) + anchored = anchor_cgt_incidence( + uk_national_frame( + person=person, + benunit=twinned.table("benunit"), + household=household, + time_period="2024", + weight_kind=uk_household_weight_kind(twinned), + household_weights=twinned.weights_for("household").values, + mass_log=twinned.mass_log, + ), + distribution=load_advani_summers_distribution(), + parameters=PARAMETERS, + ).frame + anchored_person = anchored.table("person") + by_household = pd.Series( + anchored.weights_for("household").values, + index=anchored.table("household")["household_id"], + ) + person_weight = anchored_person["person_household_id"].map(by_household) + liable_gains = anchored_person["capital_gains"] + liable = liable_gains > PARAMETERS.annual_exempt_amount + count = 0.3 * float(person_weight[liable].sum()) + total = 0.3 * float((person_weight[liable] * liable_gains[liable]).sum()) + facts = dataclasses.replace( + load_hmrc_cgt_asset_type_facts(), + table8a_taxpayers_total=count, + table8a_gains_total=total, + table8b_individuals_taxpayers=count, + table8b_individuals_gains=total, + table8b_all_taxpayers=count, + table8b_all_gains=total, + ) + with_arms = split_cgt_residential_households( + anchored, facts=facts, parameters=PARAMETERS + ).frame + return _load_tool()._drop_stacked_layers( + with_arms, [HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE] + ) + + class TestSupportSplitRecomputeAcrossGeographicClones: """The recompute counts copies on the frame's geographic clone count. The stage reads ``K`` from its input and counts ``ceil(K x w / 60)`` copies (microcosm#931); the tool reads ``K`` from the pre-split table the same way. At one clone its receipt is the one the rule before ``K`` gave. - At every ``K`` its count check holds the stored count to the family - weights' relative tolerance of 1e-9, since the fold re-sums the copies. - Within that tolerance of a multiple of 60 it cannot tell ``n`` copies + The fold re-sums the copies, so the count check never recounts it. At one + clone a family of more than three members whose clone pair sums are all + one weight ``c`` is held exactly: the stored ``n`` must be the count of a + weight that divides by ``n`` to ``c``, or to within one float of it where + fewer than four members show their clone pair untouched. Every other + family, and every family at ``K > 1``, is held to the family weights' + relative tolerance of 1e-9. That bracket opens for a family whose + source-scaled weight ``K x w`` lies within a relative 1e-9 of a multiple + of 60 (at ``K = 1``, its own weight), where it cannot tell ``n`` copies from ``n + 1``: a stored count moved on its own is left to the flags and the family weights, and a family split consistently into one copy more goes unreported. @@ -1300,9 +1445,10 @@ def test_a_count_moved_off_a_non_multiple_is_reported( def test_a_count_moved_on_an_exact_multiple_is_left_to_the_flags( self, clone_count: int ) -> None: - # Household 1 (900: 15 copies) moved to 16. K x w / 60 is exactly 15 - # at one clone and at three, and at every clone count the tolerance - # cannot tell 15 from 16 (microcosm#931 review), so the count check + # Household 1 (900: 15 copies) moved to 16 on its root alone. Its 15 + # members no longer match that count, so even at one clone the family + # is not held exactly, and K x w / 60 is exactly 15, where the bracket + # cannot tell 15 from 16 (microcosm#931 review). The count check # passes; the flags and the family weights report the family instead. _, frame = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) household = frame.table("household").copy() @@ -1319,47 +1465,202 @@ def test_a_count_moved_on_an_exact_multiple_is_left_to_the_flags( } assert problems["support_split_family_weights"] == 15 - @pytest.mark.parametrize("clone_count", [1, 3]) - def test_a_family_split_into_one_more_copy_on_a_multiple_goes_unreported( - self, clone_count: int, monkeypatch + def test_at_one_clone_a_family_split_into_one_more_copy_on_a_multiple_is_reported( + self, monkeypatch ) -> None: - # The blind spot the tolerance leaves (microcosm#931 review). A stage - # that counted floor(q) + 1 copies for q = K x w / 60 instead of - # ceil(q) differs only where q is an integer: here the four exact - # multiples (900, 1,800, 4,020 and 7,620), each split in full into one - # copy more, every member at w / (n + 1). No check reports it at any - # clone count; only the exact recount records the gap. - from microcosm.build.uk_runtime import cgt_support - - def floor_plus_one(weights, maximum_copy_weight, *, clone_count): - quotient = ( - clone_count - * np.asarray(weights, dtype=float) - / float(maximum_copy_weight) - ) - return np.maximum(np.floor(quotient).astype("int64") + 1, 1) - - with monkeypatch.context() as patched: - patched.setattr(cgt_support, "cgt_support_copy_counts", floor_plus_one) - split, frame = _clone_first_split_then_cloned( - clone_count, _NESTING_SCENARIO - ) + # The floor(q) + 1 stage splits the four exact multiples (900, 1,800, + # 4,020 and 7,620) consistently into one copy more, root and copies + # together, every member at w / (n + 1). Each family's members still + # carry one weight, and every weight that divides by n + 1 to it + # counts n copies, so the K = 1 hold reports all four. The bracket, + # which the tool before applied here, reported none. + split, frame = _split_with_floor_plus_one(monkeypatch, 1, _NESTING_SCENARIO) + + receipt, problems, exact, within = _held_families(frame, _NESTING_SCENARIO) + + assert problems == {"support_split_copies_stored": 4} + assert receipt["copies"] == split.copies_created == 259 + 4 + assert exact == receipt["families"] == 6 + assert within == 0 + + def test_beyond_one_clone_a_family_split_into_one_more_copy_on_a_multiple_goes_unreported( + self, monkeypatch + ) -> None: + # The blind spot the bracket leaves (microcosm#931 review): at three + # clones the same four families, split into one copy more in every + # clone, pass every check, since no family is held exactly at K > 1; + # only the exact recount records the gap. + split, frame = _split_with_floor_plus_one(monkeypatch, 3, _NESTING_SCENARIO) receipt, problems = _exhausting_receipt(frame, _NESTING_SCENARIO) assert problems == {} - assert receipt["copies"] == split.copies_created == clone_count * (259 + 4) + assert receipt["copies"] == split.copies_created == 3 * (259 + 4) assert receipt["copies_recomputed"] < receipt["copies"] + def test_at_one_clone_families_one_to_three_ulps_above_a_multiple_stay_green( + self, + ) -> None: + # Weights one to three ulps above a multiple of 60 count one copy more + # than the multiple, and the fold's re-sum of their copies can land + # back on the multiple, so an exact recount of the fold misses two + # copies here. Every member carries fl(w / n), so the K = 1 hold + # accepts each family's count exactly. + scenario = { + "weights": [ + _ulps_above(900.0, 1), + _ulps_above(1_800.0, 2), + _ulps_above(4_020.0, 3), + _ulps_above(7_620.0, 1), + _ulps_above(300.0, 2), + 40.0, + 9_000.0, + ], + "wealth": [7e3, 6e3, 5e3, 4e3, 3e3, 2e3, 1.0], + "incomes": [BAND_INCOMES[0]] * 6 + [BAND_INCOMES[37_700]], + "top_cells": {0: {250_000: 1_000_000.0}}, + } + split, frame = _clone_first_split_then_cloned(1, scenario) + + receipt, problems, exact, within = _held_families(frame, scenario) + + assert problems == {} + assert exact == receipt["families"] == 5 + assert within == 0 + assert receipt["copies"] == split.copies_created == 244 + assert receipt["copies_recomputed"] == 242 + + def test_at_one_clone_a_family_with_a_corrected_member_falls_to_the_bracket( + self, monkeypatch + ) -> None: + # One copy of the 900 family, which the floor(q) + 1 stage split into + # 16, carries a rounding amount more, as where an exact-total + # correction falls on a member. Its members no longer carry one + # weight, so the family is held to the bracket, which cannot tell 15 + # from 16 at the multiple; the other three stay held and reported. + split, frame = _split_with_floor_plus_one(monkeypatch, 1, _NESTING_SCENARIO) + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + (copy,) = copy_id_table.mint([1], "cgt_support_split", [1]) + weights[ids == copy] *= 1.0 + 1e-12 + + _, problems, exact, within = _held_families( + _rebuild(frame, household_weights=weights), _NESTING_SCENARIO + ) + + assert problems == {"support_split_copies_stored": 3} + assert exact == 5 + assert within == 0 + + def test_at_one_clone_a_family_of_three_or_fewer_is_left_to_the_bracket( + self, monkeypatch + ) -> None: + # Each of the three exact-total corrections after the split moves one + # row, so in a family of three or fewer every member may carry a + # corrected weight, and none vouches for w / n: the family is held to + # the bracket. Under the floor(q) + 1 stage the 120 family becomes + # three members and goes unreported; the 240 family becomes five and + # is reported. The correct stage receipts green. + assert _load_tool()._SUPPORT_SPLIT_CORRECTED_ROWS == 3 + scenario = { + "weights": [120.0, 240.0, 9_000.0], + "wealth": [7e3, 6e3, 1.0], + "incomes": [BAND_INCOMES[0]] * 2 + [BAND_INCOMES[37_700]], + "top_cells": {0: {250_000: 1_000_000.0}}, + } + _, correct = _clone_first_split_then_cloned(1, scenario) + _, mutated = _split_with_floor_plus_one(monkeypatch, 1, scenario) + + _, correct_problems = _exhausting_receipt(correct, scenario) + receipt, problems, exact, within = _held_families(mutated, scenario) + + assert correct_problems == {} + assert problems == {"support_split_copies_stored": 1} + assert receipt["families"] == 2 + assert exact == 1 + assert within == 0 + + def test_at_one_clone_members_moved_one_float_are_held_to_within_one_float( + self, + ) -> None: + # Every member of the 900 family (15 at 60) has its halves re-split + # unevenly, as the #970 anchor re-splits a trimmed pair, with its pair + # sum one float above 60, as the anchor's roundings can leave it. No + # member shows its clone pair untouched, so the hold reaches one float + # either side of the common weight; 900 divides by 15 to 60, within + # reach, and the stage's 15 stands. Held exactly, the count would be + # reported, since every weight dividing by 15 to the moved weight + # counts 16. + _, frame = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + members = np.r_[ + np.int64(1), copy_id_table.mint([1] * 14, "cgt_support_split", range(1, 15)) + ] + moved = float(np.nextafter(60.0, np.inf)) + weights[np.isin(ids, members)] = 31.0 + weights[np.isin(ids, copy_id_table.mint(members, "cgt_incidence_clone", 1))] = ( + moved - 31.0 + ) + moved_frame = _rebuild(frame, household_weights=weights) + + _, problems, exact, within = _held_families(moved_frame, _NESTING_SCENARIO) + exactly = _load_tool() + exactly._SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS = 0 + exact_problems: dict[str, object] = {} + exactly._e8_support_split( + moved_frame, + exact_problems, + distribution=support_distribution(_NESTING_SCENARIO["top_cells"]), + parameters=PARAMETERS, + permutation_seed=7, + ) + + assert problems == {} + assert exact == 5 + assert within == 1 + assert exact_problems == {"support_split_copies_stored": 1} + + def test_through_the_anchor_and_the_residential_split_the_hold_stays_sound( + self, monkeypatch + ) -> None: + # The nesting scenario through the support split, the incidence + # clone, a hand redraw giving the clone carriers gains in all four of + # the anchor's groups, the #970 anchor and the residential split, with + # the arms folded away as the E8 entry folds them. The anchor trims + # some pairs and the arms divide the liable clones, so two families + # show fewer than four untouched pairs (every carrier of the 900 + # family falls in one trimmed group) and are held to within one + # float; the other four are held exactly. The correct stage receipts + # green. Under the floor(q) + 1 stage three of the four families on a + # multiple are reported; the fourth, the 900 family, is held to + # within one float, a reach that takes in the multiple. + _, correct = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) + _, mutated = _split_with_floor_plus_one(monkeypatch, 1, _NESTING_SCENARIO) + + correct_receipt, correct_problems, correct_exact, correct_within = ( + _held_families(_through_the_residential_split(correct), _NESTING_SCENARIO) + ) + _, problems, exact, within = _held_families( + _through_the_residential_split(mutated), _NESTING_SCENARIO + ) + + assert correct_problems == {} + assert correct_receipt["families"] == 6 + assert (correct_exact, correct_within) == (4, 2) + assert problems == {"support_split_copies_stored": 3} + assert (exact, within) == (4, 2) + def test_at_one_clone_a_family_folded_just_above_a_multiple_is_green( self, ) -> None: # One copy of household 1 (900: 15 copies of 60) carries a rounding - # amount more, as where the stage's exact-total correction falls on a - # copy, so the family folds to just above 900, where an exact recount - # gives 16. The count check holds the stage's 15 to the tolerance at - # one clone too (microcosm#931 review), so the receipt is green; its - # exact recount still records the extra copy. + # amount more, as where an exact-total correction falls on a member, + # so the family folds to just above 900, where an exact recount gives + # 16. Its members no longer carry one weight, so the family is held to + # the bracket at one clone too (microcosm#931 review), which accepts + # the stage's 15, and the receipt is green; its exact recount still + # records the extra copy. split, frame = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) ids = frame.table("household")["household_id"].to_numpy() weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index ca32649fcd..7ca738a9ee 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -846,23 +846,35 @@ def _e8_support_split( ``parameters``, and the stored wealth columns. The stored layer must agree: the households the rule divides are exactly the roots of the flagged copies (the pre-clone rows carrying ``cgt_support_copies`` above - one), each family's count is ``ceil(K x w / maximum_copy_weight)``, ``K`` - the geographic clone count the stage read from the frame (microcosm#931; - the recompute reads it from the pre-split table the same way), to the - family weights' relative tolerance of 1e-9 at every ``K``. The fold - re-sums the ``n`` copies, which can land ulps from ``w``: at ``K = 1`` - for a weight a few ulps above a multiple of the maximum, or where the - stage's exact-total correction fell on a copy, and at ``K > 1`` by more - than the count's snap absorbs. So for a family whose weight lies within - that tolerance of a multiple of the maximum, the check accepts ``n`` or + one), and each family's count is ``ceil(K x w / maximum_copy_weight)``, + ``K`` the geographic clone count the stage read from the frame + (microcosm#931; the recompute reads it from the pre-split table the same + way). The fold re-sums the ``n`` copies, which can land ulps from ``w``: + at ``K = 1`` for a weight a few ulps above a multiple of the maximum, or + where an exact-total correction fell on a member, and at ``K > 1`` by + more than the count's snap absorbs. So the count is not recounted from + the fold. At ``K = 1`` a family of more than three members whose clone + pair sums are all one value ``c`` is held exactly: the stored ``n`` must + be the count of a weight ``w`` with ``fl(w / n) == c``, or, where fewer + than four of its members show their clone pair untouched, with + ``fl(w / n)`` within one float of ``c``, the most the #970 anchor and the + residential arms move an uncorrected member + (``_support_count_held_exactly``). Every other family, and every family + at ``K > 1``, is held to the family weights' relative tolerance of 1e-9: + the stored ``n`` must lie between the counts at the folded weight times + ``1 - 1e-9`` and ``1 + 1e-9``. That bracket opens for a family whose + source-scaled weight ``K x w`` lies within a relative 1e-9 of a multiple + of 60 (at ``K = 1``, its own weight), where it accepts ``n`` or ``n + 1``. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a family the stage split consistently into one copy more there, every member at ``w / (n + 1)`` - and indexed in full, is reported by no check at any ``K``. The receipt's - ``copies_recomputed`` stays the exact recount, so it can differ from - ``copies`` by one for each family whose re-summed weight lands across a - multiple, or that was split into one copy more, while the count check - passes. + and indexed in full, is reported by no check. Where the exact hold + applies, that blind spot narrows to at most the multiple itself (the + multiple and the float either side of it where ``c`` may have moved). + The receipt's ``copies_recomputed`` stays the exact recount from the + re-summed fold, so it can differ from ``copies`` by one for each family + whose re-summed weight lands across a multiple, or that was split into + one copy more, while the count check passes. Every member of a family carries ``w / n`` at stage time (its clone pair sum, since the clone runs afterwards), the copies of each root are indexed ``1..n-1`` without gaps and carry the root's count, no count is @@ -980,8 +992,9 @@ def _e8_support_split( problems["support_split_flags"] = flag_problems receipt["families"] = int(len(family_positions)) - # Fold: clones onto originals (pair sums), then copies onto roots. - pre_clone = _pre_clone_household_weights(frame) + # Fold: clones onto originals (pair sums), then copies onto roots. Which + # members' halves agree feeds the K = 1 count hold below. + pre_clone, equal_halves = _pre_clone_fold(frame) folded = pre_clone.copy() np.add.at(folded, root_positions, pre_clone[lineage.copy_positions]) @@ -1064,15 +1077,15 @@ def recompute(person_t, household_t, weights_t): agreed = (recomputed_family & stored_family).to_numpy() stored_agreed = stored_count.to_numpy()[agreed] # The fold re-sums the n copies, which can land ulps from w: at K = 1 for - # a weight a few ulps above a multiple of 60, or where the exact-total - # correction fell on a copy, and at K > 1 by more than the count's K-ulp - # snap absorbs (an exact multiple of 60 at K = 11, for one). An exact - # recount would flag a family the stage counted right, so at every K the - # count is held to the family weights' tolerance instead: the stored n + # a weight a few ulps above a multiple of 60, or where an exact-total + # correction fell on a member, and at K > 1 by more than the count's + # K-ulp snap absorbs (an exact multiple of 60 at K = 11, for one). An + # exact recount of the fold would flag a family the stage counted right, + # so the count is held to the family weights' tolerance: the stored n # must lie between the counts at the folded weight times 1 - 1e-9 and - # 1 + 1e-9 (microcosm#931 review). Within that tolerance of a multiple it - # cannot tell n from n + 1, so a family split consistently into one copy - # more there goes unreported. + # 1 + 1e-9 (microcosm#931 review). That bracket opens for a family whose + # source-scaled weight K x w lies within a relative 1e-9 of a multiple of + # 60 (at K = 1, its own weight), where it cannot tell n from n + 1. agreed_weights = pre_weights[agreed] lowest, highest = ( cgt_support_copy_counts( @@ -1082,6 +1095,21 @@ def recompute(person_t, household_t, weights_t): ) for factor in (1.0 - 1e-9, 1.0 + 1e-9) ) + if clone_count == 1: + # At K = 1 a family of more than three members that all fold to one + # weight c is held exactly instead: the stored n must be the count + # of a weight w that divides by n to c, or to within the one float + # the later stages' roundings can move c where fewer than four + # members show their pair untouched (_support_count_held_exactly). + held, held_lowest, held_highest = _support_count_held_exactly( + pre_clone, + equal_halves, + roots=np.flatnonzero(pre_split)[agreed], + stored_counts=stored_agreed, + lineage=lineage, + ) + lowest = np.where(held, held_lowest, lowest) + highest = np.where(held, held_highest, highest) count_off = int(((stored_agreed < lowest) | (stored_agreed > highest)).sum()) if count_off: problems["support_split_copies_stored"] = count_off @@ -1102,6 +1130,163 @@ def recompute(person_t, household_t, weights_t): return receipt +#: The exact-total corrections between the CGT support split's division and +#: the identity tool's read, each of which moves one household row: the +#: split's own (only where none of the whole households it tries can carry +#: it does spi_support's shared fallback put it on a root or copy), the +#: incidence clone's and the residential split's. The #970 anchor applies +#: none. So at most this many support-family members carry a corrected +#: weight, and a family of more members has one that carries none. +_SUPPORT_SPLIT_CORRECTED_ROWS = 3 +#: How many floats the #970 anchor and the residential arms' fold can move +#: the pair sum of a K = 1 family member no correction touched. +_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS = 1 + + +def _support_count_held_exactly( + pre_clone: np.ndarray, + equal_halves: np.ndarray, + *, + roots: np.ndarray, + stored_counts: np.ndarray, + lineage: _SupportCopyLineage, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """The K = 1 support families whose copy count is held exactly, and its counts. + + ``roots`` are the positions of the families the recompute and the stored + layer agree on, ``stored_counts`` their stored counts ``n``. A family is + held where all ``n`` members are present (its root and copies 1 to + ``n - 1``), ``n`` exceeds :data:`_SUPPORT_SPLIT_CORRECTED_ROWS`, and every + member's clone pair sum (``pre_clone``) is one value ``c``. The split gave + every member ``fl(w / n)``, ``w`` the family's source weight, and at most + that many members carry a weight an exact-total correction moved after + it, so at least one carries ``fl(w / n)`` as only the later stages' + roundings can have moved it: + + - Where more than that many members show equal halves + (``equal_halves``), one of those is uncorrected, and its pair sum is + exactly ``fl(w / n)``. The incidence clone halves a weight exactly. The + #970 anchor scales a pair's clone half by a factor of at most one and + adds what it took to the original. The residential split divides only + a liable clone half, which the anchor leaves at factor one, and never + an original (the clone stage gives gains to clone carriers alone). So + an uncorrected member whose halves are still equal after the anchor + and the arms' fold carries both at exactly half of ``fl(w / n)``, and + ``c == fl(w / n)``. + - Otherwise ``c`` is within :data:`_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS` + float of ``fl(w / n)``. At ``K = 1`` a member's weight lies in [32, + 64) once ``n > 3``. The anchor's three roundings move a trimmed pair's + sum by at most three quarters of an ulp before its last rounding. The + anchor refuses gains on any clone member but its carrier, so a liable + clone half becomes its own row and one arm, and the two products and + their sum keep it within one ulp of itself, half an ulp of the pair. + + A held family's stored ``n`` must then be the count + ``cgt_support_copy_counts(w', 60, clone_count=1)`` of some ``w'`` whose + quotient ``fl(w' / n)`` lies that close to ``c``. Those ``w'`` are one + run of floats, and the count rises with ``w'``, so the counts at the + run's ends bound it. A family the stage counted right always passes. One + it split consistently into one copy more or fewer passes only where that + run reaches across the count's step. By design arithmetic over every + ``n`` from 4 to 9,998, that leaves at most the multiple of 60 itself + where the family is held exactly, and the multiple and the float either + side of it where the hold reaches one float. Returns the held mask over + ``roots`` and the counts at the run's ends (zero where not held). + """ + + from microcosm.build.uk_runtime.cgt_support import ( + CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + cgt_support_copy_counts, + ) + + rows = len(pre_clone) + copy_roots = lineage.root_positions + copy_positions = lineage.copy_positions + copies_present = np.bincount(copy_roots, minlength=rows)[roots] + differing = np.bincount( + copy_roots, + weights=(pre_clone[copy_positions] != pre_clone[copy_roots]).astype(float), + minlength=rows, + )[roots] + untouched = equal_halves[roots].astype(np.int64) + np.bincount( + copy_roots, + weights=equal_halves[copy_positions].astype(float), + minlength=rows, + )[roots].astype(np.int64) + held = ( + (copies_present + 1 == stored_counts) + & (stored_counts > _SUPPORT_SPLIT_CORRECTED_ROWS) + & (differing == 0) + ) + lowest = np.zeros(len(roots), dtype=np.int64) + highest = np.zeros(len(roots), dtype=np.int64) + if held.any(): + floats = np.where( + untouched[held] > _SUPPORT_SPLIT_CORRECTED_ROWS, + 0, + _SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS, + ) + least, greatest = _floats_dividing_to( + pre_clone[roots[held]], stored_counts[held], floats + ) + lowest[held] = cgt_support_copy_counts( + least, CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=1 + ) + highest[held] = cgt_support_copy_counts( + greatest, CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=1 + ) + return held, lowest, highest + + +def _floats_dividing_to( + quotients: np.ndarray, divisors: np.ndarray, floats: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """The least and greatest float ``w`` with ``fl(w / n)`` near each ``c``. + + Near means within ``floats`` floats of ``c``. Division by a positive + ``n`` is monotone, so those ``w`` are one run of consecutive floats, + found by stepping with ``np.nextafter`` from ``fl(n x c)``. Where no + float divides to within reach (a weight no division by ``n`` gives, + which no member the split divided carries), the two floats either side + of the gap are returned instead, the lesser first. + """ + + targets = np.asarray(quotients, dtype=float) + divisors = np.asarray(divisors, dtype=float) + floats = np.asarray(floats, dtype=np.int64) + low, high = targets.copy(), targets.copy() + for step in range(int(floats.max(initial=0))): + reach = floats > step + low = np.where(reach, np.nextafter(low, -np.inf), low) + high = np.where(reach, np.nextafter(high, np.inf), high) + least = _step_while(low * divisors, lambda w: w / divisors < low, np.inf) + least = _step_while( + least, lambda w: np.nextafter(w, -np.inf) / divisors >= low, -np.inf + ) + greatest = _step_while(high * divisors, lambda w: w / divisors > high, -np.inf) + greatest = _step_while( + greatest, lambda w: np.nextafter(w, np.inf) / divisors <= high, np.inf + ) + return np.minimum(least, greatest), np.maximum(least, greatest) + + +def _step_while( + values: np.ndarray, condition: Callable[[np.ndarray], np.ndarray], toward: float +) -> np.ndarray: + """``values`` stepped one float at a time toward ``toward`` while ``condition``.""" + + values = values.copy() + for _ in range(64): + moving = condition(values) + if not moving.any(): + return values + values = np.where(moving, np.nextafter(values, toward), values) + raise RuntimeError( + "e8 identity receipt: the support-split count hold found no float " + "run within 64 steps." + ) + + def _e8_residential_split( frame, problems: dict[str, object], @@ -1800,10 +1985,17 @@ def plans_equal(left: pd.Series, right: pd.Series) -> bool: "pair sum against w / n at rtol 1e-9 / atol 1e-6 (the fold and " "the anchor each cost one float rounding generation); " "support-split mass record: new_total against old_total at rtol " - "1e-9; support-split copy counts: between the counts at the " - "folded family weight times 1 - 1e-9 and 1 + 1e-9, so within that " - "tolerance of a multiple of the maximum copy weight a family split " - "consistently into one copy more is not reported; support-split " + "1e-9; support-split copy counts: at K = 1, for a family of more " + "than three members whose clone pair sums are all one weight c, " + "exactly the count of a weight w with fl(w / n) == c, or with " + "fl(w / n) within one float of c where fewer than four members " + "show their clone pair untouched; for every other family and at " + "K > 1, between the counts at the folded family weight times " + "1 - 1e-9 and 1 + 1e-9, which for a family whose source-scaled " + "weight K x w lies within a relative 1e-9 of a multiple of 60 (at " + "K = 1, its own weight) does not report a family split " + "consistently into one copy more; copies_recomputed is the exact " + "recount from the re-summed fold; support-split " "families, flags and student_loan_plan: exact equality; " "band-donor seats, bands and " "carriers: exact equality; band-donor weights within a band and " @@ -2148,6 +2340,22 @@ def _pre_clone_household_weights(frame) -> np.ndarray: without the clone layer is returned unchanged. """ + return _pre_clone_fold(frame)[0] + + +def _pre_clone_fold(frame) -> tuple[np.ndarray, np.ndarray]: + """``_pre_clone_household_weights`` and, per row, whether its halves agree. + + The mask is true on an original whose clone carries bitwise the + original's own weight, with the residential arms folded onto both, and + on every row of an artifact without the clone layer; false on the + clones and the arms. The K = 1 count hold of ``_e8_support_split`` reads + it: the incidence clone halves each weight exactly, and halves still + equal after the #970 anchor and the arms' fold are both still exactly + half the weight the clone halved, unless an exact-total correction moved + one of them (``_support_count_held_exactly``). + """ + from microcosm.build.uk_runtime.cgt_residential_split import ( HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, ) @@ -2161,7 +2369,7 @@ def _pre_clone_household_weights(frame) -> np.ndarray: benunit = frame.table("benunit") weights = _pre_residential_household_weights(frame) if HOUSEHOLD_IS_CGT_CLONE not in household.columns: - return weights + return weights, np.ones(len(household), dtype=bool) # The residential arms (microcosm#1063) are stacked after the clone, so # the clone pairing runs on the frame without them, their mass already # folded onto their sources. @@ -2187,8 +2395,14 @@ def _pre_clone_household_weights(frame) -> np.ndarray: kept_positions = np.flatnonzero(keep) clone_positions = kept_positions[clone_kept] original_positions = kept_positions[original_kept] + equal_halves = np.zeros(len(household), dtype=bool) + # The members' weights are positive and finite, so == is bitwise + # equality on them. + equal_halves[original_positions] = ( + weights[original_positions] == weights[clone_positions] + ) weights[original_positions] += weights[clone_positions] - return weights + return weights, equal_halves def _pre_residential_household_weights(frame) -> np.ndarray: From b6e027869b9b6edd82d36e9c9b4ac86f365c3edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:57:59 +0000 Subject: [PATCH 031/108] Bind the first-party helpers outside microcosm.build into the stages that call them (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The derived binding rule followed microcosm.build only, so a stage that computed through microcosm.fit, microcosm.frame, microcosm.graph or microcosm.calibrate replayed a stale output when that code changed. As María ruled on 9 Nov, the rule now extends to every first-party package, and the core modules every node runs through are listed with reasons instead of bound. It only changes cache keys. - test_uk_graph: _HELPER_SCOPE is "microcosm.", which covers all seven workspace packages. _CORE_MODULES lists where the walk stops: the Frame (frame.bundle, schema and weights), the graph executor (graph.executor and population) and the modules that compute node keys (graph.keys, decl and kernel). Each reason says what runs through the module and what catches an edit that changes values: H1's stored bytes for the Frame; H2, which compares the graph's spine with the legacy stage plan's, for the executor's modules; every key moving where node_key or source_hash returns something else. A new test checks that the walk from run_graph meets each, that the stages' transforms meet only the Frame's three, all three together, and that every spine stage kernel declares SeedSource.NONE. - The wider walk adds one refusal-only check (frame.rules.assert_rules_engine_country) and one listed name (frame.concepts.SOURCE_HOUSEHOLD_ID_COLUMN). The names test checks that name's reason against each reader's declared columns. The dense population test stays on microcosm.build until the other UK kernels take the rule. - graph_kernels binds what the test then requires, 71 bindings in 47 stages: - fit.qrf and fit.model (_QRF_MODULES) in was_wealth, was_lisa, nts_bus_travel, lcfs_consumption, etb_vat, etb_services, property_components, uc_reporter_redraw, spi_housing_shell, spi_employment_shell and aps_occupation; - frame.materialize in frs_hmrc_spine_leaves, spi_support_channel, hmrc_spi_income_spine and spi_income_band_donors; - frame.rules in uc_take_up and pension_credit_take_up; - calibrate.geography_constants in hmrc_cgt_gains_spine, isa_subscriptions and spi_housing_shell; - graph.randomness and graph.canonical in regional_home_values; - the engine adapter, beside national_frame, in every stage's hash, since national_frame stamps each frame with its ISA income contract and savings income projection. The 38 stages that call no engine gain it, and the engine re-key test now expects every stage to move on an adapter edit. - From the 8 Nov re-review: the rung sampler declares national_sampling alone at f100, where sample_uk_spine_root returns before frame_sampling, and both below it, and the sampled-root test pins both. The spine-build install scan follows what the source adapter and the rung sampler wrap, which neither declares to the node, and requires it to come from the stage's module; the test also checks that the observation adapter forwards a wrapped declaration. - Text: the band donor result docstring says which band fields the gate reads exactly, and that carrier_regions is a count of regions no check reads. The two nesting comments and the band-donor changelog's Funding bullet add that the 1/K split rounds when K is not a power of two. The helper-bindings changelog lists what is now bound and what is listed. What moves: node keys only. graph_kernels.py, which every spine kernel hashes, changes, so every spine node re-keys, and every node downstream with it; so do the identity and claim nodes, whose kernels live there. national_sampling.py changes, so the dense population nodes re-key, and the production root now hashes the sampler's module alone at f100 and both modules below it. The spi_band_donors.py docstring re-keys stage 17 and the SPI income stage, which the above already covers. No value, declaration or generated file moves: uk_spine.json and the H2 fixture hold stage contract hashes, not code hashes, and the fixture check regenerates byte for byte; the source_stages projection and the coverage manifest are current. Tests: test_uk_graph (38 tests) and 8 related engine-free files (440 tests) pass, and ruff is clean on the edited files. Mutations under scratch, each caught: age_tail newly calling fit.qrf or frame.materialize without the binding; was_wealth losing its QRF binding; regional_home_values losing canonical; the adapter leaving the common hash; a core entry no node runs through, one only some stages compute through, canonical listed as core, and the weights entry renamed to a module the executor never reaches; the f100 and below-f100 declarations swapped; a declaring class from another module inside the source adapter (which the parent's install test passes) or inside the rung sampler. The H2 fixture check and the H2 parity test pass through the engine slot. Co-Authored-By: Claude Opus 5.5 --- .../931-band-donors-per-clone.changed.md | 2 +- changelog.d/931-uk-helper-bindings.changed.md | 33 +- .../build/uk_runtime/graph_kernels.py | 105 +++- .../build/uk_runtime/national_sampling.py | 10 +- .../build/uk_runtime/spi_band_donors.py | 7 +- .../tests/engine_free/uk/test_uk_graph.py | 457 ++++++++++++++---- .../engine_free/uk/test_uk_spi_band_donors.py | 8 +- .../tests/engine_free/uk/test_uk_spi_spine.py | 9 +- 8 files changed, 487 insertions(+), 144 deletions(-) diff --git a/changelog.d/931-band-donors-per-clone.changed.md b/changelog.d/931-band-donors-per-clone.changed.md index 5d41126774..799b10c20b 100644 --- a/changelog.d/931-band-donors-per-clone.changed.md +++ b/changelog.d/931-band-donors-per-clone.changed.md @@ -1,6 +1,6 @@ The SPI income band donors (`spi_income_band_donors`) are seated within each geographic clone (#931). Where the households carry `household_clone_index`, the stage runs its Efraimidis-Spirakis walk once in each clone, over that clone's candidates, with the same person-keyed uniforms, keys and tie order, and the same seats in every clone (María's rulings of 7 Nov). The seats and the household share's cap read the eligible households counted in distinct `source_household_id` values, so they do not depend on the clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, seats exactly the one-clone donors, carriers and bands. Each donor weighs its band's published taxpayers over the clone count times the band's donor count. A spine without the column, as every spine is today, is one clone. -- **Funding.** A donor's funding stratum is its region within its clone, a derived stratum (the manifest's `funding_strata` stays `[region]`), so every clone's regions conserve their own mass. One mass scale, the smallest any clone's region allows, serves every clone, so each band keeps one donor weight and the stage gate's and the identity tool's band-mass reconciliations hold as before. The exact-total correction still lands on one incumbent row, never on a donor, so a band's donors stay equal bit for bit at any clone count. The seating nests exactly at any clone count. Clone 0's weights and funding factors nest, its weights as the one-clone weights over the clone count, only while the common scale is the one a one-clone spine takes: one at both, or bound by a region of clone 0. Where another clone's region binds, every clone's donors, clone 0's included, take the lower scale. Even then the corrected row, which can sit in any clone, moves by a few ulps of the total, so the weights nest to rounding, not bitwise. The mass record's reason is unchanged, so neither the frame's content identity nor the coverage manifest's restated reason moves. +- **Funding.** A donor's funding stratum is its region within its clone, a derived stratum (the manifest's `funding_strata` stays `[region]`), so every clone's regions conserve their own mass. One mass scale, the smallest any clone's region allows, serves every clone, so each band keeps one donor weight and the stage gate's and the identity tool's band-mass reconciliations hold as before. The exact-total correction still lands on one incumbent row, never on a donor, so a band's donors stay equal bit for bit at any clone count. The seating nests exactly at any clone count. Clone 0's weights and funding factors nest, its weights as the one-clone weights over the clone count, only while the common scale is the one a one-clone spine takes: one at both, or bound by a region of clone 0. Where another clone's region binds, every clone's donors, clone 0's included, take the lower scale. Even while they nest, the weights nest to rounding, not bitwise: the corrected row, which can sit in any clone, moves by a few ulps of the total, and the 1/K split rounds when the clone count is not a power of two, so uncorrected rows also miss by about one ulp then. The mass record's reason is unchanged, so neither the frame's content identity nor the coverage manifest's restated reason moves. - **Receipt.** The evidence gains `clone_count`, and `funding_strata` lists `household_clone_index` where the column is present (the funding rows' labels then read `region|clone`). `expected_donor_households`, `expected_donor_count` and the donor counts cover every clone; `donor_weight` and `planned_donor_weight` are one clone's weights. Under the receipts' floor of three (`receipt_counts`), the eligible household counts, in all and per band, are counted in source households and rounded in place of the exact counts, each band's being clone 0's, which a one-clone spine counts alike; `seating_scale` reads `full` or `capped`, since an exact scale below one gives back the eligible count; a band's `carrier_mean_age` reads `<3` when its carriers come from fewer than three source households or the band seats fewer than three households per clone; and `carrier_region_counts` and the funding rows' `donor_households` are rounded. The band rows' donor and carrier counts stay exact row counts, which the gate reads. - **Gate.** The `spi_income_band_donor_support` check refuses a receipt without a positive integer `clone_count`. A donor's weight times the clone count must stay within the maximum, a band's donors must split evenly across the clones, and at full seating a band seats its plan and never fewer than the clone count times the minimum. The seating scale is read as `full` or `capped`, and a value in neither form is named by its type alone. At one clone the checks and their messages are as before, apart from the seating scale's form. - **Refusals.** A clone short of candidates for a band is refused with the number of such clones and the fewest candidates found. A non-integer, missing or negative clone index, or an index from 0 to K - 1 that no household carries, is refused by count. diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index b1da9d6f2e..7211a947d0 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,17 +1,26 @@ -Every UK spine stage now hashes each module in `microcosm.build` that its transform computes through, and so do the seven dense population kernels that share `_PopulationKernel`'s hash (#931). `source_hash` hashes only the files it is given, so a stage that called a function in a module its hash did not name replayed a stale cached output when that module changed. +Every UK spine stage now hashes each first-party module that its transform computes through, and the seven dense population kernels that share `_PopulationKernel`'s hash do so for `microcosm.build` (#931). `source_hash` hashes only the files it is given, so a stage that called a function in a module its hash did not name replayed a stale cached output when that module changed. - **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports and module attributes. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: - - three listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; + - four listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; - the stage's own declaration (`country_spec`, `source_manifest` and the spec engine), whose content the node key already hashes as the stage contract; - - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; one such name is listed. + - the listed core modules every node runs through (see Scope); + - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; two such names are listed. - A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies the rule to the seven dense population kernels, with two listed names. Two more cover the transforms the production spine build installs itself. Control tests pin each way of naming a helper that the rule follows, each kind it skips, and each way a refusal-only check could write. -- **Scope.** The rule follows `microcosm.build` only. `microcosm.graph` and `microcosm.frame` run every node, so binding them by rule would re-key every node on any edit to them. Only some value-computing modules outside `microcosm.build` are bound, by hand: `microcosm.fit`'s categorical, donor and held-out fitters in `spi_employment_shell` and `aps_occupation`, and the engine adapter in the stages that call the engine. These are still unbound: - - `microcosm.fit.qrf` and `fit.model`, which 11 stages fit through, those two among them; - - `microcosm.frame.rules` and `frame.materialize`; - - the engine adapter's helpers, in stages that call no engine; - - `microcosm.graph.randomness`, which `regional_home_values` draws through; - - `microcosm.calibrate.geography_constants`. + A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies the rule to the seven dense population kernels, still over `microcosm.build` alone, with two listed names. Two more cover the transforms the production spine build installs itself; a transform installed inside the source adapter or the rung sampler must come from its stage's module, since neither hands the node what the wrapped transform declares. Control tests pin each way of naming a helper that the rule follows, each kind it skips, the core-module stop, and each way a refusal-only check could write. +- **Scope.** The rule follows every first-party package: `microcosm-build`, `-calibrate`, `-data`, `-diagnostics`, `-fit`, `-frame` and `-graph`, all under the `microcosm.` namespace. What the stages compute through outside `microcosm.build` is now bound as it is inside it, 71 bindings in 47 stages: + - `microcosm.fit.qrf` and `fit.model` in 11 stages: the nine that fit through the regime-gated QRF, and `spi_employment_shell` and `aps_occupation`, whose classifier resolves its fit weights through `fit.model` and whose held-out scores and folds use the QRF's; + - `microcosm.frame.materialize`, whose `engine_tables` copies a frame's tables, in the three stages of `spi_spine` and the band donors; + - `microcosm.frame.rules` in `uc_take_up` and `pension_credit_take_up`, which type their engine by the adapter's `ProjectingRulesEngine`, a protocol that extends its `RulesEngine`; + - the engine adapter in every spine stage, beside `national_frame`, which stamps each frame it builds with the adapter's ISA income contract and savings income projection; 38 stages that call no engine gain it; + - `microcosm.graph.randomness` and its canonical encoder `microcosm.graph.canonical` in `regional_home_values`, which draws its keyed uniforms through them; + - `microcosm.calibrate.geography_constants`, the UK region tier, in `hmrc_cgt_gains_spine`, `isa_subscriptions` and `spi_housing_shell`. + + Listed instead, each with a reason a test checks: + - the core modules every node runs through, where the rule stops, since binding one would re-key every node on any edit to it. They are the Frame's three modules (`microcosm.frame.bundle`, `schema` and `weights`), which a stage's transform meets all three or none of; the graph executor's (`microcosm.graph.executor` and `population`); and the modules that compute node keys (`keys`, `decl` and `kernel`), which no stage's transform meets. No spine key hashes them. An edit to the Frame that changes values is left to its own tests and to H1, whose fixtures run it through `run_graph` against stored bytes. One to the executor's modules that changes a value on H2's fixture fails H2, which compares the graph's spine with the legacy stage plan's. One that changes what `node_key` or `source_hash` returns changes every key. A test checks that the walk from `run_graph` meets each of them. `microcosm.graph.canonical` also encodes every key, but `regional_home_values` draws through it, so it is bound there instead; + - `microcosm.frame.rules`' `assert_rules_engine_country`, the fourth refusal-only check, which every stage that takes an engine calls first; + - `SOURCE_HOUSEHOLD_ID_COLUMN` from `microcosm.frame.concepts`, CopyLineage's default source column, the second listed name. The support channel writes the column by its literal name. Every stage that reads the constant declares the column by that name, which its key hashes, and counts before the copies only beside a copy column, or declares neither, so a renamed constant leaves the lineage readers to refuse or count as before. + + Still outside the rule: calls through an object whose class the code never names. The engine the engine stages call is one: the hand-bound engine modules cover the adapter, the engine-block machinery and the UK declaration, but not the adapter's own imports (`policyengine_uk_concepts`, `concept_mapping`, `concepts`, `materialize` and `rules`). - **What is bound.** `national_frame` joins the modules every spine stage hashes. The stage helper table gains 231 bindings of 54 modules in 49 stages. Among them: - in the CGT and high-value stages: `property_localisation`, `cgt_structure`, `spi_support`'s exact-total helpers, `cgt_imputation`, `cgt_asset_type` and `advani_summers`; - `employer_pension_contributions` in `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`; @@ -24,6 +33,6 @@ Every UK spine stage now hashes each module in `microcosm.build` that its transf - `frs_spine`, `spi_support_channel`, `spi_income_band_donors`, `uc_deduction_attributes` and `etb_services` join the counting stages. The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input`, `gates` and `spi_support`, whose replacement strata the sample reads. -- **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. It declares both modules through `graph_implementation_dependencies`, which the root node hashes. Its behaviour is unchanged. +- **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. Through `graph_implementation_dependencies`, which the root node hashes, it declares `national_sampling` at every rung, since it resets the root's indexes at f100 too, and `frame_sampling` only below f100, where it samples. Its behaviour is unchanged. - **Other dense kernels.** The rule is not yet applied to the UK's other graph kernels. Run over them, it finds unhashed modules in the identity node (`household_lineage`, `source_lineage`), the property value and band nodes (`tenure_constants`, `source_lineage`), the private rent node (`source_lineage`, `receipt_counts`, `national_frame`) and the BRMA and RFDR nodes (`receipt_counts`). The calibration and size kernels miss `national_frame`, and the size search and refit also miss the doctrine, local target and target weight modules. The rule finds more in the gate, target, national and export kernels. Binding them is left open. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler at every rung. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 7707ce8e9f..e0dd15be9c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -26,10 +26,15 @@ import numpy as np import pandas as pd +from microcosm.calibrate import geography_constants from microcosm.fit import categorical as fit_categorical from microcosm.fit import donors as fit_donors from microcosm.fit import held_out as fit_held_out +from microcosm.fit import model as fit_model +from microcosm.fit import qrf as fit_qrf from microcosm.frame import Frame, MassChangeRecord, WeightKind +from microcosm.frame import materialize as frame_materialize +from microcosm.frame import rules as frame_rules from microcosm.frame.adapters import engine_population from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from microcosm.frame.adapters import ( @@ -49,6 +54,8 @@ load_source, source_hash, ) +from microcosm.graph import canonical as graph_canonical +from microcosm.graph import randomness as graph_randomness from microcosm.graph.population import dtype_for_token from .. import ( @@ -204,7 +211,8 @@ # shared engine-block machinery with the UK declaration it judges each batch # against (microcosm#931). An edit to any of them moves the hash of every such # stage; test_uk_graph derives the stages from their engine calls and from -# their transforms' engine parameter. +# their transforms' engine parameter. Every stage also hashes the adapter +# beside national_frame (_implementation_hash). _ENGINE_MODULES = ( uk_engine_adapter, engine_population, @@ -212,15 +220,16 @@ ) # Imported modules are not traversed by ``source_hash``, so each stage binds -# every module in microcosm.build holding code its transform computes -# through, beyond its own module (microcosm#931). test_uk_graph derives that -# set: it follows what the transform names into the module defining it, to a -# fixed point, and needs no binding for a listed refusal-only check. Where it -# stops at a single string, a column-name function or a record, the module is -# hashed by the stage or a stage upstream of it, or the name is listed with -# its reason; a collection of strings, like a number, is a value. Modules in -# the other workspace packages are bound by hand, and test_uk_graph names -# the ones that are not yet. +# every first-party module holding code its transform computes through, +# beyond its own module (microcosm#931). test_uk_graph derives that set: it +# follows what the transform names into the module defining it, in any +# workspace package, to a fixed point. It needs no binding for a listed +# refusal-only check, nor for the core modules every node runs through (the +# Frame, the graph executor and the modules that compute node keys), which it +# lists with their reasons. Where it stops at a single string, a column-name +# function or a record, the module is hashed by the stage or a stage upstream +# of it, or the name is listed with its reason; a collection of strings, like +# a number, is a value. # Every stage whose module or bound helpers import the shared keyed-draw # helpers (``microcosm.build.stochastic_assignment``), or import another # module's function that draws through them, binds them as well, so an edit @@ -233,6 +242,11 @@ # A pinned source tab is read through frs_spine's tab reader, which checks the # tab's digest through trace. _PINNED_TAB_MODULES = (frs_spine, trace) +# The QRF donor stages fit and draw through microcosm.fit's regime-gated QRF, +# which resolves its fit columns and weights through fit.model. The +# categorical fitters resolve theirs through fit.model too, and their held-out +# scores and folds use the QRF's gate calibration and folds (microcosm#931). +_QRF_MODULES = (fit_qrf, fit_model) _STAGE_HELPER_MODULES = { # The three FRS draw stages' receipts measure their take-up flags through @@ -278,13 +292,16 @@ # vendored facts; the weekly leaves annualise on frs_spine's week count # (microcosm#931). Their module, spi_spine, also mints the support # channel's copies, so the leaves and the SPI income stage bind the copy-id - # table as the support channel does. + # table as the support channel does. Its three stages, like the band + # donors, copy their frames' tables through microcosm.frame's + # engine_tables (microcosm#931). "frs_hmrc_spine_leaves": ( frs_hmrc_source, copy_id_table, employer_pension_contributions, *_FEED_FACT_MODULES, frs_spine, + frame_materialize, stochastic_assignment, ), # The annual income descriptors' receipt follows the shared disclosure rules @@ -329,6 +346,7 @@ spi_band_donors, spi_support, target_materialization, + frame_materialize, stochastic_assignment, ), # The spine's raw ids sit below the copy-id table's first field, and every @@ -359,6 +377,7 @@ frs_take_up, gates, terminal_gates, + frame_materialize, stochastic_assignment, ), # The band donors read HMRC's ITL band taxpayers as vendored facts and the @@ -369,6 +388,7 @@ *_FEED_FACT_MODULES, frs_take_up, spi_income, + frame_materialize, stochastic_assignment, ), # The exact-total correction the CGT split hands the high-value split is @@ -391,7 +411,9 @@ # correction; the support split also takes the adult age and oldest-adult # rule from cgt_structure, and the residential split its logistic from # cgt_asset_type; the incidence clone sums cgt_imputation's income proxy - # components into the income behind its propensity (microcosm#931). + # components into the income behind its propensity; the gains spine names + # HMRC's regional conditioning rows by microcosm.calibrate's UK region tier + # (microcosm#931). "cgt_support_split": ( copy_id_table, *_FEED_FACT_MODULES, @@ -431,6 +453,7 @@ "hmrc_cgt_gains_spine": ( *_FEED_FACT_MODULES, advani_summers, + geography_constants, hmrc_capital_gains, hmrc_income, target_materialization, @@ -453,10 +476,13 @@ # fit's weight kind, gate the fits and clip donor support to the donor's # range (microcosm#931); the WAS recipients' predictors map benefit units to # households, count family roles, group tenure by the shared tenure map and - # name the HBAI income predictor. + # name the HBAI income predictor. The WAS stage, and the LISA, bus, LCFS, + # ETB, landlord, reporter and housing stages below, fit through the QRF + # modules (microcosm#931). "was_wealth": ( *_ENGINE_MODULES, *_PINNED_TAB_MODULES, + *_QRF_MODULES, frs_brma, gates, household_income, @@ -475,6 +501,7 @@ hmrc_isa_statistics, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, + *_QRF_MODULES, frs_brma, gates, household_income, @@ -494,17 +521,22 @@ spi_income, spi_support, *_FEED_FACT_MODULES, + *_QRF_MODULES, frs_take_up, gates, hmrc_uprating, stochastic_assignment, ), # The regional values rank as the dense node does and read its knots - # (microcosm#1112), over the tenure map's owners (microcosm#931). + # (microcosm#1112), over the tenure map's owners; their keyed uniforms are + # microcosm.graph's, which encodes each draw's coordinates as canonical + # JSON (microcosm#931). "regional_home_values": ( property_localisation, rowwise_geography, tenure_constants, + graph_randomness, + graph_canonical, ), # The purchases read the vendored HMRC Stamp Tax rows through the feed-pinned # reader and mark SDLT's regions from the FRS region map (microcosm#1095). @@ -520,6 +552,7 @@ bus_use_incidence, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, + *_QRF_MODULES, gates, household_income, qrf_gates, @@ -539,6 +572,7 @@ energy_pricing, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, + *_QRF_MODULES, bus_fare_pricing, donor_uprating, gambling_participation, @@ -556,6 +590,7 @@ *_ENGINE_MODULES, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, + *_QRF_MODULES, donor_uprating, gates, household_income, @@ -569,6 +604,7 @@ *_ENGINE_MODULES, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, + *_QRF_MODULES, bus_fare_pricing, bus_support_pricing, donor_uprating, @@ -586,6 +622,7 @@ "uc_reporter_redraw": ( uc_relationships, *_ENGINE_MODULES, + *_QRF_MODULES, qrf_gates, uc_capital_coherence, stochastic_assignment, @@ -618,10 +655,12 @@ # attributes, the take-up population and the family-type roles, and counts # its pools and receipt on the source lineage (microcosm#1334); the # attributes map benefit units to households and judge their realisation - # by the shared tolerance, and the year's rules come from the FRS release - # (microcosm#931). + # by the shared tolerance, and the year's rules come from the FRS release; + # it types its engine as the adapter's ProjectingRulesEngine, a protocol + # that extends microcosm.frame.rules' RulesEngine (microcosm#931). "uc_take_up": ( *_ENGINE_MODULES, + frame_rules, *_FEED_FACT_MODULES, legacy_closure_take_up, uc_deduction_attributes, @@ -643,10 +682,12 @@ ), # The Pension Credit redraw reuses the UC stage's household-to-benefit-unit # weight mapping and the adapter's engine materialization; it reads the - # take-up contract and the year's rules (microcosm#931). + # take-up contract and the year's rules, and types its engine by the same + # protocol as the UC redraw (microcosm#931). "pension_credit_take_up": ( uc_capital_coherence, *_ENGINE_MODULES, + frame_rules, frs_release, take_up_contract, stochastic_assignment, @@ -674,21 +715,30 @@ # The stored attendance is one engine materialization (microcosm#1095), # whose year's rules come from the FRS release (microcosm#931). "private_school_attendance": (*_ENGINE_MODULES, frs_release), - # The housing shell keeps council tax bands within the nation and gates - # its fits; the deduction attributes map benefit units to households and + # The housing shell keeps council tax bands within the nation, whose + # regions are microcosm.calibrate's UK region tier, and gates its fits; + # the deduction attributes map benefit units to households and # judge their realisation by the shared tolerance; the salary sacrifice # reads its bonus anchor as vendored facts and gates its fits; the # student loans resolve their year's rules (microcosm#931). - "spi_housing_shell": (council_tax_constants, qrf_gates, stochastic_assignment), + "spi_housing_shell": ( + council_tax_constants, + geography_constants, + *_QRF_MODULES, + qrf_gates, + stochastic_assignment, + ), "uc_deduction_attributes": (frs_brma, stage_health, stochastic_assignment), "salary_sacrifice": (*_FEED_FACT_MODULES, qrf_gates, stochastic_assignment), "student_loans": (frs_release, stochastic_assignment), # The subscriptions walk the CGT bands on its income proxy (with the # proxy's loss relief), read HMRC's ISA tables through their typed loader # and round their receipts' support (microcosm#1415); the loss relief - # evaluates its named term, and the donor tabs are pinned (microcosm#931). + # evaluates its named term, the donor tabs are pinned, and the regions are + # microcosm.calibrate's UK region tier (microcosm#931). "isa_subscriptions": ( cgt_imputation, + geography_constants, hmrc_income, hmrc_isa_statistics, receipt_counts, @@ -739,11 +789,13 @@ # The employment shell fits, scores and draws through microcosm.fit's # categorical classifier, held-out scores and keyed donors, reads the # shared occupation codes, and counts its receipt's support by the shared - # disclosure rules (microcosm#1490). + # disclosure rules (microcosm#1490); the classifier and held-out scores + # compute through the QRF modules (microcosm#931). "spi_employment_shell": ( fit_categorical, fit_donors, fit_held_out, + *_QRF_MODULES, occupation_constants, receipt_counts, stochastic_assignment, @@ -752,10 +804,12 @@ # classifier and held-out scores, reads the shared occupation codes, and # counts its receipt's support by the shared disclosure rules # (microcosm#1490); it reads its donor tab through trace's digest and - # records its fit's weight kind (microcosm#931). + # records its fit's weight kind, and its classifier and held-out scores + # compute through the QRF modules (microcosm#931). "aps_occupation": ( fit_categorical, fit_held_out, + *_QRF_MODULES, occupation_constants, receipt_counts, gates, @@ -861,13 +915,16 @@ def _implementation_hash(kernel: object, stage: str, transform: object | None) - # still declare extra behavior-bearing sources through # ``graph_implementation_dependencies``. Every kernel rebuilds its frame # on national_frame's schema, and the transforms build, validate and read - # the time period of their frames through it (microcosm#931). + # the time period of their frames through it (microcosm#931). It stamps + # each frame it builds with the engine adapter's ISA income contract and + # savings income projection, so every kernel hashes the adapter with it. dependencies = getattr(transform, "graph_implementation_dependencies", None) return source_hash( type(kernel), stage_evidence, graph_evidence, national_frame, + uk_engine_adapter, _stage_artifacts, _mass_log_payload, _stage_module(stage), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py index 825b58fa7e..842344c9aa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py @@ -20,7 +20,8 @@ sampling seed that the UK builds share, and the spine build's root sampler: a spine built below f100 samples its FRS root household by household (:func:`sample_uk_spine_root`) through :class:`UKSampledSpineRootTransform`, -which the root node hashes with this module and the shared core. +which the root node hashes with this module, and below f100 with the shared +core too. """ from __future__ import annotations @@ -330,7 +331,10 @@ class UKSampledSpineRootTransform: The spine build wraps the FRS root's transform in it. Its sampling code lives here and in the shared core, not in the root stage's module, so it - declares both for the root node to hash (microcosm#931). + declares them for the root node to hash (microcosm#931): this module at + every rung, since it resets the root's indexes at f100 too, and the shared + core only below f100, since :func:`sample_uk_spine_root` returns before + calling it at f100. """ def __init__( @@ -348,6 +352,8 @@ def __init__( def graph_implementation_dependencies(self) -> tuple[object, ...]: """The modules the root node hashes for this adapter's sampling.""" + if self.fraction == 1.0: + return (sys.modules[__name__],) return (sys.modules[__name__], frame_sampling) def _sample(self, assembled): diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py index 69cf857909..2fe980f063 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py @@ -224,9 +224,10 @@ class UKSPIIncomeBandDonorResult: (clone 0's, which a one-clone spine counts alike), the funding rows' ``donor_households`` and ``carrier_region_counts``. A band row's ``carrier_mean_age`` is withheld below three carrier source households or - three seats per clone. The band rows' ``donor_households``, ``carriers`` - and ``carrier_regions`` and ``donor_count`` stay exact row counts, which - the stage gate reads exactly; ``expected_donor_count`` is the plan's. + three seats per clone. The band rows' ``donor_households`` and + ``carriers``, and ``donor_count``, stay exact row counts, which the stage + gate reads exactly; a band row's ``carrier_regions`` is an exact count of + regions that no check reads. ``expected_donor_count`` is the plan's. """ frame: Frame diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 84f18252dc..92c3902b5e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -782,9 +782,11 @@ def test_every_stage_that_calls_the_engine_binds_the_engine_modules() -> None: than one clone, is the adapter's, the shared engine-block machinery's and the UK declaration's (microcosm#931). ``source_hash`` does not follow imports, so every stage whose module calls the engine, or whose transform - takes one, binds all three, and no other stage binds any. The one - exception takes the engine only to check its country and computes - nothing through it.""" + takes one, binds all three, and no other stage binds any of them in the + stage table. The one exception takes the engine only to check its country + and computes nothing through it. Every stage also hashes the adapter + beside national_frame, which stamps its frames with the adapter's ISA + income contract and savings income projection (the next test).""" from microcosm.build.uk_runtime import graph_kernels engine_modules = set(graph_kernels._ENGINE_MODULES) @@ -831,11 +833,17 @@ def test_the_engine_call_scan_sees_any_receiver(tmp_path) -> None: @pytest.mark.parametrize( - "module_name", - ["policyengine_uk", "engine_population", "policyengine_uk_population"], + ("module_name", "every_stage"), + [ + # national_frame stamps every frame through the adapter's ISA income + # contract and savings income projection, so every stage hashes it. + ("policyengine_uk", True), + ("engine_population", False), + ("policyengine_uk_population", False), + ], ) def test_engine_module_changes_invalidate_every_stage_that_calls_the_engine( - monkeypatch, module_name + monkeypatch, module_name, every_stage ): import importlib @@ -871,7 +879,7 @@ def changed_module_bytes(path): for ref in affected: assert registry.implementation_hash(ref) != before[ref], ref for ref in controls: - assert registry.implementation_hash(ref) == before[ref], ref + assert (registry.implementation_hash(ref) != before[ref]) is every_stage, ref # The rule behind the bindings above, for every helper (microcosm#931): @@ -881,20 +889,19 @@ def changed_module_bytes(path): # kernel, names, and follow each named definition into the module that # defines it, through re-exports and module attributes, to a fixed point. -#: The package the rule follows: microcosm.build holds every UK stage and its -#: helpers. The workspace's other packages stay outside it. microcosm.graph -#: and microcosm.frame are the executor and the Frame that every node runs -#: on, so binding them by rule would re-key every node of every graph on any -#: edit to them. Value-computing code out there is bound only by hand: -#: microcosm.fit's categorical, donor and held-out fitters in -#: spi_employment_shell and aps_occupation, and the engine adapter in the -#: stages that call the engine. The rest is not bound yet: microcosm.fit.qrf -#: and fit.model, which the QRF donor stages and those two fit through; -#: microcosm.frame.rules and frame.materialize; the engine adapter's helpers -#: in stages that call no engine; microcosm.graph.randomness, which -#: regional_home_values draws through; and -#: microcosm.calibrate.geography_constants. -_HELPER_SCOPE = "microcosm.build." +#: The packages the rule follows: every first-party package, each under the +#: microcosm namespace (microcosm-build, -calibrate, -data, -diagnostics, +#: -fit, -frame and -graph). It binds what the stages compute through there +#: as in microcosm.build: microcosm.fit's QRF, model, categorical, donor and +#: held-out modules; microcosm.frame's materialize and rules modules and the +#: engine adapter, which national_frame stamps every frame through; +#: microcosm.graph's keyed uniforms and their canonical encoder; and +#: microcosm.calibrate's region tier. It stops at the core modules every node +#: runs through (``_CORE_MODULES``). The dense population kernels stay on +#: microcosm.build (``_DENSE_SCOPE``) until the UK's other graph kernels take +#: the rule. +_HELPER_SCOPE = "microcosm." +_DENSE_SCOPE = "microcosm.build." #: Where the rule stops. What a stage reads from these is its own #: declaration, which its node key already hashes as the stage contract: its #: operations, artifacts and resource pins (graph._stage_contract_sha256). @@ -903,6 +910,60 @@ def changed_module_bytes(path): "microcosm.build.source_manifest", "microcosm.build.spec_engine", ) +#: The Frame's reason, which its three modules share. +_FRAME_REASON = ( + "the spine stages' transforms compute on it, every spine stage kernel " + "builds one, and the executor holds every population version in one. No " + "spine key hashes it, so an edit that changes values is left to its own " + "tests and to H1, whose fixtures run it through run_graph against stored " + "bytes" +) +#: The core modules every node runs through, which the rule neither follows +#: nor asks a stage to bind, each with what runs through it and what catches +#: an edit to it that would change values. Binding one would re-key every +#: node on any edit to it. The Frame's are the only ones a stage's transform +#: names; the executor and the modules that compute node keys run every node +#: from outside it. microcosm.graph.canonical also encodes every key, but +#: regional_home_values draws through it, so that stage binds it instead. +#: test_the_core_modules_are_on_every_nodes_path checks the table. +_CORE_MODULES: dict[str, str] = { + "microcosm.frame.bundle": f"the Frame: {_FRAME_REASON}", + "microcosm.frame.schema": f"the Frame's entity schema: {_FRAME_REASON}", + "microcosm.frame.weights": ( + f"the Frame's typed weights and mass records: {_FRAME_REASON}" + ), + "microcosm.graph.executor": ( + "runs every node (run_graph): keys it, projects its declared inputs, " + "calls its kernel and applies its result. No spine key hashes it, so " + "an edit that changes a value on H2's fixture fails H2, which compares " + "every cell of the spine run through it with the legacy stage plan's, " + "which does not run through it" + ), + "microcosm.graph.population": ( + "the executor applies every node's result to its population version " + "through it (patch); as for the executor, H2 catches an edit that " + "changes a value on its fixture" + ), + "microcosm.graph.kernel": ( + "every node's inputs and outputs pass through its KernelContext and " + "KernelResult, where H2 catches a changed value as for the executor; " + "every spine stage kernel's implementation hash is computed through " + "its source_hash, so an edit that changes what that returns changes " + "every key" + ), + "microcosm.graph.keys": ( + "computes every node's key and its artifacts' keys (node_key), so an " + "edit that changes what it returns changes every key; no spine stage " + "kernel reads the generator the executor seeds from a key (each " + "declares SeedSource.NONE)" + ), + "microcosm.graph.decl": ( + "node_key hashes every node's normative declaration and resolves its " + "inputs and population version through the compiled graph, as the " + "executor does, so an edit that changes how a node is declared or " + "what it reads changes its key" + ), +} #: Checks that read values only to decide whether to refuse, each with why it #: cannot change what its caller computes. The rule does not follow them, and #: a definition that names nothing else of value needs no binding for them; @@ -924,6 +985,11 @@ def changed_module_bytes(path): "returns its argument unchanged and refuses anything but person, " "benunit or household; the support channel's column names call it" ), + ("microcosm.frame.rules", "assert_rules_engine_country"): ( + "reads the engine's declared country and refuses one that declares " + "none or another; the stages that take an engine call it before " + "computing" + ), } #: Names a stage reads from a module that neither its node nor any node #: upstream of it hashes, each with why an edit to them cannot leave the @@ -938,6 +1004,16 @@ def changed_module_bytes(path): "expand writes; no spine stage writes them (checked), so a spine " "frame never carries them and their name moves no spine value" ), + ("microcosm.frame.concepts", "SOURCE_HOUSEHOLD_ID_COLUMN"): ( + "names the source household column, CopyLineage's default source, " + "which the support channel writes by its literal name. A node is " + "given only its declared columns, so under a renamed constant the " + "lineage readers would find no source column, which they refuse " + "unless counting before the copies on a frame without a copy column. " + "A spine stage that reads it declares the column by that name, which " + "its key hashes, and counts before the copies only beside a copy " + "column, or declares neither, and counts as before (checked)" + ), } #: The same for the dense population kernels, which run outside the spine #: graph. @@ -1298,20 +1374,33 @@ def _classify_helper(module: str, name: str, scope: str) -> str: return "value" -def _helper_walk( - root: tuple[str, str], scope: str = _HELPER_SCOPE -) -> tuple[dict[str, frozenset[str]], frozenset[tuple[str, str]]]: - """What ``root`` can compute through, and where the rule stops. +class _Walk(NamedTuple): + """What a root can compute through (``_helper_walk``).""" + + reach: dict[str, frozenset[str]] + stops: frozenset[tuple[str, str]] + core: frozenset[str] + - The first part maps every module holding a definition that can change - what ``root`` computes to those definitions: ``root`` itself, and each - ``value`` definition it names, and they name, to a fixed point. The second +def _helper_walk( + root: tuple[str, str], scope: str = _HELPER_SCOPE, *, follow_core: bool = False +) -> _Walk: + """What ``root`` can compute through, where the rule stops, and the core + modules it meets. + + ``reach`` maps every module holding a definition that can change what + ``root`` computes to those definitions: ``root`` itself, and each + ``value`` definition it names, and they name, to a fixed point. ``stops`` is every ``name``, ``column name`` or ``record`` those definitions name, - which the rule does not follow. The declaration modules and the listed - refusal-only checks are neither followed nor returned.""" + which the rule does not follow. ``core`` is every listed core module + (``_CORE_MODULES``) whose definitions they name, which the rule neither + follows nor returns in ``reach``, unless ``follow_core``. The declaration + modules and the listed refusal-only checks are neither followed nor + returned.""" reached: dict[str, set[str]] = {} stops: set[tuple[str, str]] = set() + core: set[str] = set() pending, seen = [root], set() while pending: item = pending.pop() @@ -1322,6 +1411,10 @@ def _helper_walk( if module.startswith(_DECLARATION_MODULES): continue if item != root: + if module in _CORE_MODULES: + core.add(module) + if not follow_core: + continue kind = _helper_kind(module, name, scope) if kind != "value": if kind != "check": @@ -1330,9 +1423,10 @@ def _helper_walk( reached.setdefault(module, set()).add(name) node = _module_definitions(module, scope).definitions[name] pending.extend(_named(module, node, scope)) - return ( + return _Walk( {module: frozenset(names) for module, names in reached.items()}, frozenset(stops), + frozenset(core), ) @@ -1342,7 +1436,7 @@ def _helper_reach( """Every module holding a definition that can change what ``root`` computes, with those definitions (``_helper_walk``).""" - return _helper_walk(root, scope)[0] + return _helper_walk(root, scope).reach def _stage_transforms() -> dict[str, tuple[str, str]]: @@ -1415,10 +1509,11 @@ def test_every_stage_hashes_each_helper_its_transform_computes_through() -> None The rule starts at the transform graph_kernels builds for the stage and follows every definition it names, annotations included (an injected - contract or table is called through them), into the module defining it. - It does not follow a listed refusal-only check or a stage declaration, - and stops at a single string, a column-name function or a record (see - ``_helper_kind``), whose module the next test accounts for. Anything + contract or table is called through them), into the module defining it, + in any first-party package. It does not follow a listed refusal-only + check, a stage declaration or a listed core module every node runs + through, and stops at a single string, a column-name function or a record + (see ``_helper_kind``), whose module the next test accounts for. Anything else is a value, a collection of strings and a number included, and needs its module bound. The rule starts at the transform, not at the kernel that runs it: the kernel's code, in graph_kernels, which every @@ -1433,9 +1528,9 @@ def test_every_stage_hashes_each_helper_its_transform_computes_through() -> None walks = _stage_walks() assert set(walks) == set(graph_kernels._STAGE_MODULES) unhashed = { - stage: sorted(set(reach) - hashed) - for stage, (_, hashed, (reach, _)) in walks.items() - if set(reach) - hashed + stage: sorted(set(walk.reach) - hashed) + for stage, (_, hashed, walk) in walks.items() + if set(walk.reach) - hashed } assert not unhashed, unhashed @@ -1469,11 +1564,11 @@ def upstream(node_id: str) -> set[str]: return seen unlisted: dict[tuple[str, str], list[str]] = {} - for stage, (node_id, hashed, (_, stops)) in walks.items(): + for stage, (node_id, hashed, walk) in walks.items(): covered = set(hashed).union( *(hashed_by_node[item] for item in upstream(node_id)) ) - for module, name in stops: + for module, name in walk.stops: if module not in covered: unlisted.setdefault((module, name), []).append(stage) assert set(unlisted) == set(_STAGE_UNBOUND_NAMES), sorted( @@ -1488,6 +1583,84 @@ def upstream(node_id: str) -> set[str]: entity, rowwise_dataset.ladder_clone_index_column(entity), ) not in written + # The source household column's reason. The support channel writes it by + # its literal name; a stage that reads it declares it by that name and + # counts before the copies only beside a copy column, or declares neither + # it nor a copy column. + from microcosm.build.uk_runtime import spi_support + from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE + from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN + + assert f'"{SOURCE_HOUSEHOLD_ID_COLUMN}"' in Path(spi_support.__file__).read_text() + copy_columns = set(UK_COPY_LINEAGE.copy_columns) + readers = unlisted[("microcosm.frame.concepts", "SOURCE_HOUSEHOLD_ID_COLUMN")] + counting = {stage: _counts_before_copies(walks[stage][2]) for stage in readers} + assert any(counting.values()) + for stage in readers: + read = { + column + for slice_ in graph.node(walks[stage][0]).inputs + if slice_.entity == "household" + for column in slice_.columns + } + if SOURCE_HOUSEHOLD_ID_COLUMN not in read: + assert not read & copy_columns, stage + elif not read & copy_columns: + assert not counting[stage], (stage, sorted(counting[stage])) + + +def _counts_before_copies(walk: _Walk) -> set[tuple[str, str]]: + """The definitions a walk reaches that count sources before the copies: + each passes ``before_copies=True`` to a lineage reader, which then counts + a household as its own source where the frame carries neither the source + column nor a copy column.""" + + import ast + + found = set() + for module, names in walk.reach.items(): + definitions = _module_definitions(module, _HELPER_SCOPE).definitions + for name in names: + if any( + isinstance(sub, ast.keyword) + and sub.arg == "before_copies" + and isinstance(sub.value, ast.Constant) + and sub.value.value is True + for sub in ast.walk(definitions[name]) + ): + found.add((module, name)) + return found + + +def test_the_core_modules_are_on_every_nodes_path() -> None: + """Each listed core module is on every node's path, as its reason says, + and is no stage's own helper (microcosm#931). The walk from run_graph, + which keys, runs and patches every node, meets each. The core modules a + stage's transform meets are the Frame's, all three or none, so the table + exempts no module that only some stages compute through; no stage's + transform meets the executor or a key module, whose reasons rest on their + running every node from outside it. The key module's reason holds too: + every spine stage kernel declares SeedSource.NONE.""" + from microcosm.graph import SeedSource + + executor = _helper_walk(("microcosm.graph.executor", "run_graph"), follow_core=True) + on_path = set(executor.reach) | set(executor.core) + assert set(_CORE_MODULES) <= on_path, sorted(set(_CORE_MODULES) - on_path) + frame = { + module for module in _CORE_MODULES if module.startswith("microcosm.frame.") + } + walks = _stage_walks() + met = {stage: walk.core for stage, (_, _, walk) in walks.items()} + assert set().union(*met.values()) == frame + uneven = { + stage: sorted(core) for stage, core in met.items() if core and core != frame + } + assert not uneven, uneven + graph = uk_spine_graph() + registry = uk_registry(graph=graph) + for stage, (node_id, _, _) in walks.items(): + kernel = registry.get(graph.node(node_id).kernel) + assert kernel.capabilities.seed_source is SeedSource.NONE, stage def test_the_refusal_only_helpers_return_and_write_nothing() -> None: @@ -1536,8 +1709,10 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( annotation (a record's field type included) is reached, as are a numeric constant, a collection of strings and a function that writes in place or only refuses but is not listed; a listed check, a single string, a column - name, a record of plain fields, and a helper the root never names are - not, and the walk returns the names it stopped at.""" + name, a record of plain fields, a listed core module, what only a core + module names, and a helper the root never names are not. The walk returns + the names it stopped at and the core modules it met, and follows a core + module only when asked to.""" import uuid scratch = f"helper_rule_{uuid.uuid4().hex[:8]}" @@ -1586,12 +1761,20 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( " value: float\n" ), "unused": "def never(x):\n return x - 1\n", + "behind": "def kept(x):\n return x * 4\n", + "core": ( + "from .behind import kept\n\n" + "class Container:\n" + " def grow(self, x):\n" + " return kept(x)\n" + ), "stage": ( "from . import attribute\n" "from .alias import scale\n" "from .codes import CODES, OWNER\n" "from .collaborator import Contract\n" "from .computed import TABLE\n" + "from .core import Container\n" "from .direct import draw\n" "from .fielded import Inputs\n" "from .guard import guard\n" @@ -1611,6 +1794,7 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( " rate = self.contract.rate(column('household'))\n" " rate += self.inputs.rate.value()\n" " owners = [OWNER[code] for code in CODES]\n" + " rate += Container().grow(rate)\n" " return attribute.share(scale(rate)) + TABLE['a'] + FLOOR, (\n" " record, COLUMN, owners\n" " )\n\n" @@ -1624,7 +1808,8 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( monkeypatch.setitem( _REFUSAL_ONLY_HELPERS, (f"{prefix}harmless", "check"), "control" ) - reach, stops = _helper_walk((f"{prefix}stage", "Transform"), scope=prefix) + monkeypatch.setitem(_CORE_MODULES, f"{prefix}core", "control") + reach, stops, core = _helper_walk((f"{prefix}stage", "Transform"), scope=prefix) assert {module.removeprefix(prefix): names for module, names in reach.items()} == { "stage": {"Transform"}, "direct": {"draw"}, @@ -1644,6 +1829,12 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( ("harmless", "Record"), ("harmless", "column"), } + assert core == {f"{prefix}core"} + followed = _helper_walk( + (f"{prefix}stage", "Transform"), scope=prefix, follow_core=True + ) + assert set(followed.reach) - set(reach) == {f"{prefix}core", f"{prefix}behind"} + assert followed.core == core def test_the_dense_population_kernels_hash_each_helper_they_compute_through() -> None: @@ -1654,7 +1845,8 @@ def test_the_dense_population_kernels_hash_each_helper_they_compute_through() -> rule stops at outside it is listed in ``_DENSE_UNBOUND_NAMES`` with its reason (microcosm#931). The rule starts at each kernel's class; the spine gate admission check the sample kernel calls is refusal-only and is not - followed.""" + followed. It follows microcosm.build only (``_DENSE_SCOPE``) until the + UK's other graph kernels take the first-party rule.""" import inspect from microcosm.build.uk_runtime import graph_population, rowwise_geography @@ -1679,11 +1871,11 @@ def test_the_dense_population_kernels_hash_each_helper_they_compute_through() -> hashed = hashed_module_names(kernel.__new__(kernel), graph_population) # The dense clones are minted by rowwise_geography's helpers. assert rowwise_geography.__name__ in hashed - reach, stops = _helper_walk((graph_population.__name__, kernel.__name__)) - missing = sorted(set(reach) - hashed) + walk = _helper_walk((graph_population.__name__, kernel.__name__), _DENSE_SCOPE) + missing = sorted(set(walk.reach) - hashed) if missing: unhashed[kernel.ref] = missing - unlisted.update(item for item in stops if item[0] not in hashed) + unlisted.update(item for item in walk.stops if item[0] not in hashed) assert not unhashed, unhashed assert unlisted == set(_DENSE_UNBOUND_NAMES), sorted( unlisted ^ set(_DENSE_UNBOUND_NAMES) @@ -1697,16 +1889,38 @@ def test_the_dense_population_kernels_hash_each_helper_they_compute_through() -> "builds the stage's own transform from the node's declared sources and calls it" ), ("microcosm.build.observation", "ObservedTransform"): ( - "times the wrapped transform's call and records its frames' sizes" + "times the wrapped transform's call and records its frames' sizes, and " + "forwards every other attribute, the declared modules included (checked)" ), } +#: Adapters whose first argument is the stage's own transform, or builds it, +#: and which do not hand the node what that transform declares: the source +#: adapter builds it only when the node runs, after the node is keyed, and the +#: rung sampler declares its own modules alone. What either wraps must live in +#: the stage's module, which the node hashes. +_SPINE_BUILD_WRAPPERS = frozenset( + { + ("microcosm.build.uk_runtime.spine_build", "_GraphSourceTransform"), + ("microcosm.build.uk_runtime.national_sampling", "UKSampledSpineRootTransform"), + } +) -def _spine_build_installs() -> list[tuple[str | None, tuple[str, str]]]: +class _Install(NamedTuple): + """One class or factory the spine build installs (``_spine_build_installs``).""" + + stage: str | None + found: tuple[str, str] + wrapper: tuple[str, str] | None + + +def _spine_build_installs() -> list[_Install]: """Each class or factory the production spine build installs as a - stage's transform, as ``(stage, (module, name))``, seen through the - adapters above and the local names it assigns; ``None`` for a wrapper it - installs around every stage.""" + stage's transform, seen through the adapters above, the local names it + assigns and the stage entries a wrapper reads back. ``stage`` is ``None`` + for an adapter it installs around every stage; ``wrapper`` is the + innermost wrapping adapter (``_SPINE_BUILD_WRAPPERS``) the install sits + inside, or ``None``.""" import ast @@ -1720,29 +1934,52 @@ def _spine_build_installs() -> list[tuple[str | None, tuple[str, str]]]: if isinstance(node, ast.FunctionDef) and node.name == "prepare_uk_spine_execution" ) - local: dict[str, list] = {} - for sub in ast.walk(function): + + def entry(target) -> str | None: if ( - isinstance(sub, ast.Assign) - and len(sub.targets) == 1 - and isinstance(sub.targets[0], ast.Name) + isinstance(target, ast.Subscript) + and isinstance(target.value, ast.Name) + and target.value.id == "implementations" + and isinstance(target.slice, ast.Constant) ): - local.setdefault(sub.targets[0].id, []).append(sub.value) - installs: list[tuple[str | None, tuple[str, str]]] = [] + return target.slice.value + return None - def follow(expression, stage, seen=frozenset()) -> None: + local: dict[str, list] = {} + entries: dict[str, list] = {} + for sub in ast.walk(function): + if not isinstance(sub, ast.Assign) or len(sub.targets) != 1: + continue + target = sub.targets[0] + if isinstance(target, ast.Name): + local.setdefault(target.id, []).append(sub.value) + if target.id == "implementations" and isinstance(sub.value, ast.Dict): + for key, value in zip(sub.value.keys, sub.value.values, strict=True): + entries.setdefault(key.value, []).append(value) + elif entry(target) is not None: + entries.setdefault(entry(target), []).append(sub.value) + installs: list[_Install] = [] + + def follow(expression, stage, wrapper=None, seen=frozenset()) -> None: if isinstance(expression, ast.Lambda): - follow(expression.body, stage, seen) + follow(expression.body, stage, wrapper, seen) elif isinstance(expression, ast.Name) and expression.id not in seen: for value in local.get(expression.id, ()): - follow(value, stage, seen | {expression.id}) + follow(value, stage, wrapper, seen | {expression.id}) + elif entry(expression) is not None and ("entry", entry(expression)) not in seen: + # A wrapper reads back the stage's entry: what was installed there + # before it now sits inside it. + for value in entries.get(entry(expression), ()): + follow(value, stage, wrapper, seen | {("entry", entry(expression))}) elif isinstance(expression, ast.Call) and isinstance(expression.func, ast.Name): found = _resolve(module, expression.func.id, _HELPER_SCOPE) if found is None: return - installs.append((stage, found)) - if found in _SPINE_BUILD_ADAPTERS and stage and expression.args: - follow(expression.args[0], stage, seen) + installs.append(_Install(stage, found, wrapper)) + adapts = found in _SPINE_BUILD_ADAPTERS or found in _SPINE_BUILD_WRAPPERS + if adapts and stage and expression.args: + inner = found if found in _SPINE_BUILD_WRAPPERS else wrapper + follow(expression.args[0], stage, inner, seen) for sub in ast.walk(function): if isinstance(sub, ast.Assign) and len(sub.targets) == 1: @@ -1755,13 +1992,8 @@ def follow(expression, stage, seen=frozenset()) -> None: follow(value, key.value) elif isinstance(sub.value, ast.DictComp): follow(sub.value.value, None) - elif ( - isinstance(target, ast.Subscript) - and isinstance(target.value, ast.Name) - and target.value.id == "implementations" - and isinstance(target.slice, ast.Constant) - ): - follow(sub.value, target.slice.value) + elif entry(target) is not None: + follow(sub.value, entry(target)) return installs @@ -1772,49 +2004,84 @@ def test_every_transform_the_spine_build_installs_is_hashed_with_its_stage() -> installs for a stage is defined in the stage's module, declares the modules it computes through (``graph_implementation_dependencies``, which the stage's node hashes), or is an adapter listed with why it computes no - value (microcosm#931).""" + value. Inside a wrapping adapter, which hands the node nothing the wrapped + transform declares, only the stage's own module's classes and the listed + adapters may be installed (microcosm#931).""" import importlib - from microcosm.build.uk_runtime import graph_kernels + from microcosm.build.observation import ObservedTransform + from microcosm.build.uk_runtime import graph_kernels, national_sampling installs = _spine_build_installs() - assert {stage for stage, _ in installs if stage} == set( + assert {install.stage for install in installs if install.stage} == set( graph_kernels._STAGE_MODULES ) - assert {found for stage, found in installs if stage is None} == { + assert {install.found for install in installs if install.stage is None} == { ("microcosm.build.observation", "ObservedTransform") } + # Both wrappers wrap something: every source adapter wraps its stage's + # transform, and the rung sampler wraps the root's source adapter. + assert {install.wrapper for install in installs} - {None} == _SPINE_BUILD_WRAPPERS unhashed = [] - for stage, (module, name) in installs: - if (module, name) in _SPINE_BUILD_ADAPTERS or ( - stage and module == graph_kernels._stage_module(stage).__name__ - ): + for stage, (module, name), wrapper in installs: + own = bool(stage) and module == graph_kernels._stage_module(stage).__name__ + if own or (module, name) in _SPINE_BUILD_ADAPTERS: continue declared = getattr( getattr(importlib.import_module(module), name), "graph_implementation_dependencies", None, ) - if not callable(declared): - unhashed.append((stage, f"{module}.{name}")) + if wrapper is not None or not callable(declared): + unhashed.append((stage, f"{module}.{name}", wrapper)) assert not unhashed, unhashed - - -def test_the_sampled_spine_root_hashes_what_it_samples_through() -> None: - """The spine build wraps the root's transform in the rung sampler, which - samples the root below f100 and whose code lives in national_sampling and - the shared core; the root node then hashes every module the rule finds - the sampler computing through (microcosm#931).""" + # The observation adapter hands the node the declared modules of what it + # wraps, the rung sampler's among them. + sampler = national_sampling.UKSampledSpineRootTransform(None, fraction=0.1, seed=1) + observed = ObservedTransform( + sampler, stage_id="frs_spine", produced_column_count=0, observer=None + ) + assert hashed_module_names( + graph_kernels.UKCreateKernel(observed), graph_kernels + ) == hashed_module_names(graph_kernels.UKCreateKernel(sampler), graph_kernels) + + +def test_the_sampled_spine_root_hashes_what_it_samples_through(monkeypatch) -> None: + """The spine build wraps the root's transform in the rung sampler, whose + code lives in national_sampling and the shared core. The root node hashes + national_sampling at every rung, since the sampler resets the root's + indexes at f100 too, and the shared core only below f100, where + sample_uk_spine_root calls it; there it hashes every module the rule + finds the sampler computing through (microcosm#931).""" from microcosm.build import frame_sampling from microcosm.build.uk_runtime import graph_kernels, national_sampling - sampler = national_sampling.UKSampledSpineRootTransform(None, fraction=0.1, seed=1) plain = hashed_module_names(graph_kernels.UKCreateKernel(), graph_kernels) - hashed = hashed_module_names(graph_kernels.UKCreateKernel(sampler), graph_kernels) - assert hashed - plain == {national_sampling.__name__, frame_sampling.__name__} - reach = _helper_reach((national_sampling.__name__, type(sampler).__name__)) + hashed = {} + for fraction in (1.0, 0.1): + sampler = national_sampling.UKSampledSpineRootTransform( + None, fraction=fraction, seed=1 + ) + hashed[fraction] = hashed_module_names( + graph_kernels.UKCreateKernel(sampler), graph_kernels + ) + assert hashed[1.0] - plain == {national_sampling.__name__} + assert hashed[0.1] - plain == {national_sampling.__name__, frame_sampling.__name__} + reach = _helper_reach((national_sampling.__name__, "UKSampledSpineRootTransform")) assert {national_sampling.__name__, frame_sampling.__name__} <= set(reach) - assert set(reach) <= hashed, sorted(set(reach) - hashed) + assert set(reach) <= hashed[0.1], sorted(set(reach) - hashed[0.1]) + + # At f100 the root returns before the shared core samples anything. + def refuse(*args, **kwargs): + raise AssertionError("f100 must not sample") + + for name in ("sample_frame_households", "normalize_sampled_household_mass"): + monkeypatch.setattr(national_sampling, name, refuse) + frame = object() + assert national_sampling.sample_uk_spine_root(frame, fraction=1.0, seed=1) == ( + frame, + None, + ) def test_uk_graph_json_round_trip_is_canonical() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index 566cde2811..4916927047 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -1386,9 +1386,11 @@ def test_clone_zero_seats_what_one_clone_seats_and_every_clone_takes_the_seats( # Clone 0's weights, its donors' included, are the one clone's over the # clone count while the common mass scale is the one clone's, as here, - # where it is one at both. The exact-total correction moves one incumbent - # row by a few ulps of the total, in whichever clone it lands, so on these - # small frames to rtol 1e-12, not bit for bit. + # where it is one at both. On these small frames they nest to rtol 1e-12, + # not bit for bit: the exact-total correction moves one incumbent row by a + # few ulps of the total, in whichever clone it lands, and the 1/K split + # rounds when K is not a power of two, as 3 and 7 are not, so uncorrected + # rows also miss by about one ulp. weights_once = _weights_by_id(one.frame) clone_zero = _clone_of(many).loc[lambda clones: clones.eq(0)].index assert set(clone_zero) == set(weights_once.index) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py index 3440e03f08..3e3ec49e9d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py @@ -1255,10 +1255,11 @@ def copied(household, clone: int) -> set[int]: target=0.25, rows=rows, sigma=4.0 ) - # Nested weights: K x every clone-0 row is the one-clone row. The - # exact-total correction moves one row by a few ulps of the total, at one - # clone and at K, and not always in clone 0, so on these small frames to - # rtol 1e-12, not bit for bit. + # Nested weights: K x every clone-0 row is the one-clone row, on these + # small frames to rtol 1e-12, not bit for bit. The exact-total correction + # moves one row by a few ulps of the total, at one clone and at K, and not + # always in clone 0, and the 1/K split rounds when K is not a power of + # two, as 3 and 7 are not, so uncorrected rows also miss by about one ulp. one_weights = pd.Series( np.asarray(one.weights_for("household").values), index=one_household["household_id"].to_numpy(), From 2addcb6917583708dfb2163affe1b290d96bf45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:35:35 +0000 Subject: [PATCH 032/108] Hold the other UK graph kernels to the binding rule (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The derived binding rule covered the spine stages' transforms and, over microcosm.build alone, the dense population kernels. The UK's other graph kernels hashed hand-written module lists, and the 8 Nov reviews found some computing through modules they did not hash, so a node could replay a stale output after such a module changed. As María ruled on 9 Nov, the other kernels take the same rule, keys only. Nine kernels need a ruling first and stay held, each listed with its reason. - test_uk_graph: a new test enumerates every kernel the UK registries register, 99 by ref: the spine's with its gate battery; the full build's with its gate batteries and its terminal, certification and inspection kernels; and the national line's. Another checks they are every kernel class that uk_runtime and the shared atomic geography define. For each kernel the rule walks what the kernel runs, at the first-party scope: its methods in method resolution order, without the implementation hash, which computes the key, and without listed refusal-only methods. It checks the modules it meets against what the hash really hashes, recorded at every source_hash call in whichever module makes it. The stage rule's tables apply, and a name's module must be hashed by the kernel or listed: the LAD23 names' pin moves from the dense table. Three kernel stops are listed, each with a reason a test checks: - the H2 fixture's transform resolver, which names only the transform builder the stage rule starts from; - spi_spine's SPI support mass reason, which _minimal_frame reads only inside its branch on hmrc_spi_income_spine, whose node hashes its reach; - UK_GATE_REGISTRY. The bound spine reaches it only through _manifest_digests, which returns the battery run's three digests. The national gates evaluate through it, and their evaluators are held with the gate batteries (below). The dense population test folds into the new one, with its pins. A control test pins the walk: an inherited method followed, an override without super() not, super() reaching the base, the hash and a listed refusal-only method left out, a listed stop reported with what names it, and the recorder seeing a source_hash call in another module. - Admission reads. The export description and the rotated holdout now call graph_terminal._require_full_gate_admission in place of their inline decode-and-refuse, with the same messages. It and the dense solve's _admit join the refusal-only checks. Without them, the two kernels and the dense solve would hash calibration_run, full_gates and graph_evidence for a gate report they read only to refuse. - Bindings, by kernel: - the dense population kernels' shared hash (sample, normalize, expand, locations, geography mapping, pool, gate): the engine adapter, graph.canonical and graph.weight_update; - uk.full.identity: household_lineage, source_lineage, frame.concepts, graph.canonical; - uk.full.property.value: source_lineage, frame.concepts, rowwise_dataset, tenure_constants, graph.randomness, graph.canonical; - uk.full.property.band: tenure_constants; - uk.full.geography.brma and uk.full.geography.rfdr: receipt_counts; - uk.full.private_rent: source_lineage, frame.concepts, rowwise_dataset, national_frame, receipt_counts, graph.randomness, graph.canonical; - uk.identity and uk.claim: national_frame; - the four atomic geography kernels' shared hash: graph.store, through load_source_bytes' store, and frame.materialize; - the seven calibration and size kernels' shared hash: national_frame, calibrate.hierarchy, calibrate.monetary_binding, graph.canonical, and for the size search and refit target_weights, local_doctrine, local_targets, national_doctrine and calibrate.geography_constants; - uk.full.bound_spine: frame.materialize, the engine adapter, content_identity, hmrc_uprating, gate_battery, gates, trace, graph_kernels (its create-frame normalisation), graph.canonical; - uk.full-export.prepare: gates, atomic_area_support, graph_population, hmrc_uprating, local_authority_input, rowwise_dataset, rowwise_geography, frame.materialize, graph.canonical; the inspection export the same, and geography_ladder, national_frame and weight_year; - uk.full-export.readback: artifact_files, trace, geography_ladder, the engine adapter, frame.materialize, graph.canonical; - uk.full-export.package: artifact_files, trace, graph.canonical; - uk.full.certification-readiness: artifact_files, trace, gate_battery, gates, graph_evidence, calibration_run, graph.canonical; - uk.full.rotated-holdout: the 35 modules its problem reconstruction, folds and rowwise solves compute through, microcosm.calibrate's solver modules, gate_battery and stage_evidence among them; - uk.full.national-gates: gates, calibrate's artifacts, matrix, registry, target, hierarchy, monetary_binding and solve, graph_targets, graph_population, national_frame, etb_services, cgt_projection, donor_uprating, graph.canonical; - uk.full.national-readback: artifact_files, trace, frame.materialize, the engine adapter, geography_ladder, graph_population, hmrc_uprating, graph.canonical. - Held for a ruling (_HELD_KERNELS): - uk.spine-gates, uk.full-gates.preflight and uk.full-gates. Each battery shares one key across phases that declare different gates. Binding what one phase's evaluators reach would re-key the assembled gate, or the preflight and with it the dense solve, as the re-review of #1420 (E) and #1301 and #1306 kept out. - The target compilation, selection, measures and problem kernels, and the national targets and problem. test_uk_target_compile_modules holds their keys to microcosm#1497's import record, which may only shrink. Binding what the rule finds would add 253 unhashed import edges across their three groups and remove 79. - test_uk_target_compile_modules: the hash inputs gain the seven modules graph_national now imports only inside the national gates' and readback's hashes (gates, trace, donor_uprating, etb_services, calibrate.monetary_binding, the engine adapter, frame.materialize). - The helper-bindings changelog takes nit (b) of the 8 Nov re-review. Its open-kernel list becomes what this commit binds and holds. What moves: node keys only. No value, declaration or generated file moves. The 35 kernels bound above re-key. So do every spine stage and the create kernel, since graph_kernels.py changed; uk.full.geography.local_authority (graph_population.py); the full build's preflight and calibrated gates (graph_terminal.py); and the national targets and problem (graph_national.py). The target compilation, selection, measures and problem kernels and the spine gates keep their hashes. Graph nodes that re-key: every spine node and every full-graph node downstream of it; the national line from its root, uk.full.spine_checkpoint; and the transport graphs' atomic geography nodes and what they feed. uk_spine.json and the H2 fixture hold stage contract hashes, and the fixture check regenerates byte for byte; the source_stages projection and the coverage manifest are current. Tests: test_uk_graph (41 tests) and test_uk_target_compile_modules (52) pass, as do 29 related engine-free files (1,041 tests) and the engine terminal hash tests through the engine slot; ruff is clean on the edited files. Mutations under scratch, each caught: the BRMA kernel newly reading tenure_constants, the package inventory newly calling calibrate.solve, and the stage kernel's run newly calling cgt_projection, all unhashed; the RFDR hash losing receipt_counts, the calibration hash the canonical encoder, the atomic geography hash the content store, the national readback trace, and uk.identity national_frame; either admission read leaving the refusal-only table, and _admit writing its argument; _minimal_frame reading the SPI mass reason outside its stage branch; the bound spine naming the gate registry itself; the fixture resolver stop dropped; a kernel class no registry registers; and a held kernel's ref leaving the hold. The H2 fixture check regenerates byte for byte and the H2 parity test passes, both through the engine slot. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-helper-bindings.changed.md | 35 +- .../microcosm/build/graph_atomic_geography.py | 7 + .../build/uk_runtime/full_certification.py | 22 +- .../microcosm/build/uk_runtime/graph_build.py | 26 +- .../build/uk_runtime/graph_calibration.py | 27 + .../build/uk_runtime/graph_kernels.py | 14 + .../build/uk_runtime/graph_national.py | 83 ++- .../build/uk_runtime/graph_population.py | 77 +- .../build/uk_runtime/graph_terminal.py | 176 ++++- .../build/uk_runtime/inspection_export.py | 37 +- .../tests/engine_free/uk/test_uk_graph.py | 667 ++++++++++++++++-- .../uk/test_uk_target_compile_modules.py | 23 + 12 files changed, 1099 insertions(+), 95 deletions(-) diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 7211a947d0..2897a84d4a 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,12 +1,12 @@ -Every UK spine stage now hashes each first-party module that its transform computes through, and the seven dense population kernels that share `_PopulationKernel`'s hash do so for `microcosm.build` (#931). `source_hash` hashes only the files it is given, so a stage that called a function in a module its hash did not name replayed a stale cached output when that module changed. +Every UK spine stage now hashes each first-party module that its transform computes through, and every other UK graph kernel, but nine held for a ruling, each one its own code computes through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. - **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports and module attributes. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: - - four listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; + - six listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels); - the stage's own declaration (`country_spec`, `source_manifest` and the spec engine), whose content the node key already hashes as the stage contract; - the listed core modules every node runs through (see Scope); - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; two such names are listed. - A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies the rule to the seven dense population kernels, still over `microcosm.build` alone, with two listed names. Two more cover the transforms the production spine build installs itself; a transform installed inside the source adapter or the rung sampler must come from its stage's module, since neither hands the node what the wrapped transform declares. Control tests pin each way of naming a helper that the rule follows, each kind it skips, the core-module stop, and each way a refusal-only check could write. + A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies it to the UK's other graph kernels (see Kernels). Two more cover the transforms the production spine build installs itself; a transform installed inside the source adapter or the rung sampler must come from its stage's module, since neither hands the node what the wrapped transform declares. Control tests pin each way of naming a helper that the rule follows, each kind it skips, the core-module stop, and each way a refusal-only check could write. - **Scope.** The rule follows every first-party package: `microcosm-build`, `-calibrate`, `-data`, `-diagnostics`, `-fit`, `-frame` and `-graph`, all under the `microcosm.` namespace. What the stages compute through outside `microcosm.build` is now bound as it is inside it, 71 bindings in 47 stages: - `microcosm.fit.qrf` and `fit.model` in 11 stages: the nine that fit through the regime-gated QRF, and `spi_employment_shell` and `aps_occupation`, whose classifier resolves its fit weights through `fit.model` and whose held-out scores and folds use the QRF's; - `microcosm.frame.materialize`, whose `engine_tables` copies a frame's tables, in the three stages of `spi_spine` and the band donors; @@ -34,5 +34,30 @@ Every UK spine stage now hashes each first-party module that its transform compu The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input`, `gates` and `spi_support`, whose replacement strata the sample reads. - **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. Through `graph_implementation_dependencies`, which the root node hashes, it declares `national_sampling` at every rung, since it resets the root's indexes at f100 too, and `frame_sampling` only below f100, where it samples. Its behaviour is unchanged. -- **Other dense kernels.** The rule is not yet applied to the UK's other graph kernels. Run over them, it finds unhashed modules in the identity node (`household_lineage`, `source_lineage`), the property value and band nodes (`tenure_constants`, `source_lineage`), the private rent node (`source_lineage`, `receipt_counts`, `national_frame`) and the BRMA and RFDR nodes (`receipt_counts`). The calibration and size kernels miss `national_frame`, and the size search and refit also miss the doctrine, local target and target weight modules. The rule finds more in the gate, target, national and export kernels. Binding them is left open. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. +- **Kernels.** A second test applies the rule to every kernel the UK registries register but the nine held below: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; and the national gates and readback. It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at three listed kernel stops, each with a reason a test checks: + - the hermetic fixture's transform resolver, since the stage rule holds each stage's transform; + - `_minimal_frame`'s rebuild of the SPI support allocation's mass record, which only `hmrc_spi_income_spine` takes, and whose reach that stage's node hashes; + - `UK_GATE_REGISTRY` (see Held for a ruling). + + The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: + - the seven dense population kernels' shared hash: the engine adapter, the canonical encoder and the weight-update receipt; + - the geography identity: `household_lineage`, `source_lineage`, `microcosm.frame.concepts` and the canonical encoder; + - property value: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `tenure_constants`, the keyed uniforms and the canonical encoder; + - property band: `tenure_constants`; BRMA and RFDR: `receipt_counts`; + - private rent: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `national_frame`, `receipt_counts`, the keyed uniforms and the canonical encoder; + - the spine's identity and claim kernels: `national_frame`; + - the four atomic geography kernels' shared hash: the content store, which `load_source_bytes` takes, and `frame.materialize`; + - the seven calibration and size kernels' shared hash: `national_frame`, `calibrate.hierarchy`, `calibrate.monetary_binding` and the canonical encoder, and, for the size search and refit, `target_weights`, `local_doctrine`, `local_targets`, `national_doctrine` and `calibrate.geography_constants`; + - the bound spine: `frame.materialize`, the engine adapter, `content_identity`, `hmrc_uprating`, `gate_battery`, `gates`, `trace`, `graph_kernels` (its create-frame normalisation) and the canonical encoder; + - the export description: `gates`, `atomic_area_support`, `graph_population`, `hmrc_uprating`, `local_authority_input`, `rowwise_dataset`, `rowwise_geography`, `frame.materialize` and the canonical encoder. The inspection export binds the same modules, and `geography_ladder`, `national_frame` and `weight_year` too; + - the export readback: `artifact_files`, `trace`, `geography_ladder`, the engine adapter, `frame.materialize` and the canonical encoder; the package inventory: `artifact_files`, `trace` and the canonical encoder; + - the certification: `artifact_files`, `trace`, `gate_battery`, `gates`, `graph_evidence`, `calibration_run` and the canonical encoder; + - the rotated holdout: the 35 modules that its problem reconstruction, folds and rowwise solves compute through. They are the local surface's modules, `holdout`, `local_target_census`, `size_checkpoint`, `solve_progress`, `weighted_integrity`, microcosm.calibrate's solver modules, `gate_battery`, `stage_evidence`, the engine adapter and the canonical encoder; + - the national gates: `gates`, microcosm.calibrate's `artifacts`, `matrix`, `registry`, `target`, `hierarchy`, `monetary_binding` and `solve`, `graph_targets`, `graph_population`, `national_frame`, `etb_services`, `cgt_projection`, `donor_uprating` and the canonical encoder; + - the national readback: `artifact_files`, `trace`, `frame.materialize`, the engine adapter, `geography_ladder`, `graph_population`, `hmrc_uprating` and the canonical encoder. + + The LAD23 names' pin stays listed, now for every kernel that reads it. `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside the national gates' and readback's hashes. +- **Held for a ruling.** Nine kernels stay outside the rule, each listed with its reason: + - the spine gates, and the full build's preflight and calibrated gates. Each battery shares one key across phases that declare different gates, which it evaluates through `UK_GATE_REGISTRY`, where the rule stops for every kernel. Binding what one phase's evaluators reach would re-key the assembled spine gate, or the preflight and with it the dense solve, on edits that cannot change them. The national gates' evaluators are held with them; + - the target compilation, selection, measures and problem kernels, and the national targets and problem. microcosm#1497's import record holds their keys and may only shrink. Binding what the rule finds would add to it every unhashed import of each module bound. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport registry's geography nodes. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py index 5d42252363..cc631c1795 100644 --- a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py +++ b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py @@ -44,12 +44,19 @@ class _GeographyKernel(KernelBase): def implementation_hash(self) -> str: + from microcosm.frame import materialize as frame_materialize + from microcosm.graph import store as graph_store + + # load_source_bytes takes a content store, whose frame encoding writes + # through frame_materialize (microcosm#931). return source_hash( type(self), geography, canonical_json, keyed_uniform, load_source_bytes, + graph_store, + frame_materialize, dependencies=_DEPENDENCIES, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py index c336198395..26de2c75c0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py @@ -401,9 +401,27 @@ class UKFullCertificationKernel(KernelBase): def implementation_hash(self): from microcosm.data import contract - from . import full_gates, graph_terminal + from .. import artifact_files, gate_battery, gates, trace + from . import calibration_run, full_gates, graph_evidence, graph_terminal - return source_hash(sys.modules[__name__], contract, full_gates, graph_terminal) + # The readiness also computes through the comparison files' records + # (artifact_files, trace's file digest), the phase reports it decodes + # (gate_battery, gates), the spine gate manifest and its scope + # (graph_evidence, calibration_run) and its report's canonical + # encoding (microcosm#931). + return source_hash( + sys.modules[__name__], + contract, + full_gates, + graph_terminal, + artifact_files, + trace, + gate_battery, + gates, + graph_evidence, + calibration_run, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: sources = {} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py index 54152fec59..c7b9fe4ba3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -388,8 +388,32 @@ class UKBoundSpineKernel(KernelBase): ) def implementation_hash(self) -> str: + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import gate_battery, gates, trace + from . import content_identity, graph_kernels, hmrc_uprating + + # The node also computes through the H5 it loads (frame_materialize, + # the engine adapter's contracts), the checkpoint's content identity + # and pinned engine (content_identity, hmrc_uprating), the gate + # digests it derives through a battery run (gate_battery, gates, + # trace), graph_kernels' create-frame normalisation and the canonical + # encoding of its artifacts (microcosm#931). return source_hash( - type(self), national_frame, calibration_run, release_certification + type(self), + national_frame, + calibration_run, + release_certification, + frame_materialize, + uk_engine_adapter, + content_identity, + hmrc_uprating, + gate_battery, + gates, + trace, + graph_kernels, + canonical_json, ) def run(self, context: KernelContext) -> KernelResult: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py index c188dc79b4..7bf9d83c49 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py @@ -267,6 +267,24 @@ def restore_uk_graph_result( class _CalibrationKernel(KernelBase): def implementation_hash(self): + import microcosm.calibrate.geography_constants as geography_constants + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.monetary_binding as monetary_binding + + from . import ( + local_doctrine, + local_targets, + national_doctrine, + national_frame, + target_weights, + ) + + # Every solve kernel shares this hash. They also compute through the + # frame they rebuild on national_frame's schema, the targets' + # hierarchy and monetary checks in microcosm.calibrate, the canonical + # encoding of their artifacts, and the size search's and refit's + # target weights with the doctrine, local target areas and UK-wide + # geography they read (microcosm#931); test_uk_graph derives the set. return hashlib.sha256( canonical_json( { @@ -280,6 +298,15 @@ def implementation_hash(self): select_exact_k, HardConcrete, contribution_initialization, + national_frame, + calibrate_hierarchy, + monetary_binding, + canonical_json, + target_weights, + local_doctrine, + local_targets, + national_doctrine, + geography_constants, dependencies=self.capabilities.dependencies, ), } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index e0dd15be9c..96e8835234 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -1742,6 +1742,13 @@ class UKIdentityKernel(KernelBase): ref = "uk.identity@1" capabilities = _FILTER + def implementation_hash(self) -> str: + # The person ids it keeps are read on national_frame's schema + # (microcosm#931). + return source_hash( + type(self), national_frame, dependencies=self.capabilities.dependencies + ) + def run(self, context: KernelContext) -> KernelResult: person = context.tables[UK_NATIONAL_SCHEMA.person_entity] id_column = UK_NATIONAL_SCHEMA.person_id_column @@ -1755,6 +1762,13 @@ class UKClaimKernel(KernelBase): ref = "uk.claim@1" capabilities = _COMPUTE + def implementation_hash(self) -> str: + # The claimed cells are indexed by national_frame's schema's id + # columns (microcosm#931). + return source_hash( + type(self), national_frame, dependencies=self.capabilities.dependencies + ) + def run(self, context: KernelContext) -> KernelResult: columns = { (owned.entity, owned.column): pd.Series( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index c8a574003f..08f36ad832 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -1005,9 +1005,38 @@ class UKNationalGateKernel(_NationalKernel): ) def implementation_hash(self) -> str: - from .. import gate_battery - from . import battery_bindings, diagnostics, receipt_counts, weighted_integrity + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.matrix as calibrate_matrix + import microcosm.calibrate.monetary_binding as monetary_binding + import microcosm.calibrate.registry as calibrate_registry + import microcosm.calibrate.solve as calibrate_solve + import microcosm.calibrate.target as calibrate_target + + from .. import gate_battery, gates + from . import ( + battery_bindings, + cgt_projection, + diagnostics, + donor_uprating, + etb_services, + graph_population, + graph_targets, + national_frame, + receipt_counts, + weighted_integrity, + ) + # Outside the gate registry's evaluators (held for a ruling; see + # test_uk_graph's _KERNEL_STOPS) the battery also computes through + # the shared gates' evaluators (gates), the solved result and + # registry it decodes (microcosm.calibrate's artifacts, matrix, + # registry, target and hierarchy, graph_targets), the frame it + # rebuilds (graph_population, national_frame), the manifest's + # effective sample size (calibrate.solve), the admin totals' derived + # anchors (etb_services), the CGT projection (cgt_projection, + # donor_uprating) and the canonical encoding of its artifacts + # (microcosm#931). return hashlib.sha256( canonical_json( { @@ -1022,6 +1051,21 @@ def implementation_hash(self) -> str: receipt_counts, diagnostics, support_floor_rules, + gates, + calibrate_artifacts, + calibrate_matrix, + calibrate_registry, + calibrate_target, + calibrate_hierarchy, + monetary_binding, + calibrate_solve, + graph_targets, + graph_population, + national_frame, + etb_services, + cgt_projection, + donor_uprating, + canonical_json, ), "country_resources": load_country_spec("uk").fingerprint, } @@ -1126,9 +1170,40 @@ class UKNationalReadbackKernel(KernelBase): capabilities = Capabilities(Determinism.DETERMINISTIC, role=KernelRole.GATE) def implementation_hash(self) -> str: - from . import content_identity, national_frame, weight_year + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import artifact_files, trace + from . import ( + content_identity, + geography_ladder, + graph_population, + hmrc_uprating, + national_frame, + weight_year, + ) - return source_hash(type(self), content_identity, national_frame, weight_year) + # The readback also computes through the file record it takes + # (artifact_files, trace's file digest), the H5 it reads back + # (frame_materialize, the engine adapter's contracts), the export + # frame's area codes (geography_ladder), the frame it rebuilds + # (graph_population), the engine's weight projection factor + # (hmrc_uprating's pins) and the canonical encoding of its report + # (microcosm#931). + return source_hash( + type(self), + content_identity, + national_frame, + weight_year, + artifact_files, + trace, + frame_materialize, + uk_engine_adapter, + geography_ladder, + graph_population, + hmrc_uprating, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: from ..artifact_files import file_artifact diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index b40855fa2a..106fcedd27 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -200,14 +200,18 @@ def _declared_mass(before: Frame, after: Frame) -> dict: class _PopulationKernel(KernelBase): def implementation_hash(self) -> str: + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + from . import graph_evidence # The dense population nodes also compute through what these call: # the sample's household draw and mass normalisation (frame_sampling), # its strata columns (spi_support's replacement strata), the clones' # minted ids and copied tables (rowwise_geography), the national frame - # they rebuild, the local authority names the geography resolves, and - # the gate's results (microcosm#931); test_uk_graph derives the set + # they rebuild and the engine adapter's contracts it stamps on them, + # the local authority names the geography resolves, the gate's + # results, their artifacts' canonical encoding and the normalisation's + # weight-update receipt (microcosm#931); test_uk_graph derives the set # from what the kernels name. return source_hash( type(self), @@ -221,6 +225,9 @@ def implementation_hash(self) -> str: national_frame, rowwise_geography, spi_support, + uk_engine_adapter, + canonical_json, + weight_update_receipt, ) @@ -433,8 +440,23 @@ class UKGeographyIdentityKernel(_PopulationKernel): capabilities = Capabilities(Determinism.DETERMINISTIC) def implementation_hash(self) -> str: + from microcosm.frame import concepts as frame_concepts + + from .. import source_lineage + from . import household_lineage + + # The clone paths are read through the lineage readers, whose default + # source column frame_concepts names, and the draw keys are encoded + # canonically (microcosm#931). return source_hash( - type(self), atomic_household_identity, atomic_area_support, rowwise_dataset + type(self), + atomic_household_identity, + atomic_area_support, + rowwise_dataset, + household_lineage, + source_lineage, + frame_concepts, + canonical_json, ) def run(self, context: KernelContext) -> KernelResult: @@ -489,13 +511,27 @@ class UKPropertyValueKernel(_PopulationKernel): ) def implementation_hash(self) -> str: - from . import household_lineage + from microcosm.frame import concepts as frame_concepts + from .. import source_lineage + from . import household_lineage, tenure_constants + + # The spine keys come through the lineage readers and the clone index + # column rowwise_dataset names, the owner mask through + # tenure_constants' tenure categories and the owner uniforms through + # keyed draws; the spine keys and the quantile artifact are encoded + # canonically (microcosm#931). return source_hash( type(self), property_localisation, household_lineage, atomic_household_identity, + source_lineage, + frame_concepts, + rowwise_dataset, + tenure_constants, + keyed_uniform, + canonical_json, ) def run(self, context: KernelContext) -> KernelResult: @@ -587,7 +623,11 @@ class UKPropertyBandKernel(_PopulationKernel): ) def implementation_hash(self) -> str: - return source_hash(type(self), property_localisation) + from . import tenure_constants + + # property_localisation's owner mask reads tenure_constants' tenure + # categories (microcosm#931). + return source_hash(type(self), property_localisation, tenure_constants) def run(self, context: KernelContext) -> KernelResult: household = context.tables["household"] @@ -706,7 +746,10 @@ class UKBRMAAreaKernel(_PopulationKernel): ) def implementation_hash(self) -> str: - return source_hash(type(self), brma_area_layer) + from . import receipt_counts + + # The receipt rounds its counts in receipt_counts (microcosm#931). + return source_hash(type(self), brma_area_layer, receipt_counts) def run(self, context: KernelContext) -> KernelResult: household = context.tables["household"] @@ -759,7 +802,10 @@ class UKRFDRAreaKernel(_PopulationKernel): ) def implementation_hash(self) -> str: - return source_hash(type(self), rfdr_area_layer) + from . import receipt_counts + + # The receipt rounds its counts in receipt_counts (microcosm#931). + return source_hash(type(self), rfdr_area_layer, receipt_counts) def run(self, context: KernelContext) -> KernelResult: household = context.tables["household"] @@ -819,14 +865,29 @@ class UKPrivateRentKernel(_PopulationKernel): ) def implementation_hash(self) -> str: - from . import household_lineage + from microcosm.frame import concepts as frame_concepts + + from .. import source_lineage + from . import household_lineage, receipt_counts + # The spine keys come through the lineage readers and the clone index + # column rowwise_dataset names, and are encoded canonically; the + # tie-breaks are keyed draws; the receipt counts source households on + # national_frame's schema and rounds them in receipt_counts + # (microcosm#931). return source_hash( type(self), private_rent_localisation, property_localisation, household_lineage, atomic_household_identity, + source_lineage, + frame_concepts, + rowwise_dataset, + national_frame, + receipt_counts, + keyed_uniform, + canonical_json, ) def run(self, context: KernelContext) -> KernelResult: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 15f0f6e85b..8e18f8d313 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -358,18 +358,49 @@ class UKExportPrepareKernel(KernelBase): ) def implementation_hash(self) -> str: + from microcosm.frame import materialize as frame_materialize + + from .. import gates + from . import ( + atomic_area_support, + graph_population, + hmrc_uprating, + local_authority_input, + rowwise_dataset, + rowwise_geography, + ) + + # The descriptor also computes through the ladder gate's results and + # region codes (gates, rowwise_geography, local_authority_input), the + # area code frames (atomic_area_support), the engine tables + # (frame_materialize) and the clone index columns rowwise_dataset + # names, the engine's weight projection factor (hmrc_uprating's + # pins), the frame it rebuilds (graph_population) and the canonical + # encoding of the descriptor (microcosm#931). The gate report it reads + # only admits it (_require_full_gate_admission). return source_hash( - sys.modules[__name__], national_frame, geography_ladder, weight_year + sys.modules[__name__], + national_frame, + geography_ladder, + weight_year, + gates, + atomic_area_support, + graph_population, + hmrc_uprating, + local_authority_input, + rowwise_dataset, + rowwise_geography, + frame_materialize, + canonical_json, ) def run(self, context: KernelContext) -> KernelResult: for value in context.artifacts.values(): if value.type == FULL_GATE_REPORT_TYPE: - _, enforcement = decode_full_gate_report(value.payload) - if not enforcement["artifact_permitted"]: - raise ValueError( - "UK export preparation refused by a structural full-build gate." - ) + _require_full_gate_admission( + value.payload, + "UK export preparation refused by a structural full-build gate.", + ) bindings = json.loads(str(context.params["bindings"])) bindings["artifacts"] = { name: value.key for name, value in context.artifacts.items() @@ -392,7 +423,26 @@ class UKExportReadbackKernel(KernelBase): ) def implementation_hash(self) -> str: - return source_hash(sys.modules[__name__], national_frame) + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import artifact_files, trace + + # The readback also computes through the file record it takes + # (artifact_files, trace's file digest), the H5 it reads back + # (frame_materialize's table reader, the engine adapter's contracts), + # the area code columns it checks (geography_ladder) and the + # canonical encoding of its report (microcosm#931). + return source_hash( + sys.modules[__name__], + national_frame, + artifact_files, + trace, + frame_materialize, + uk_engine_adapter, + geography_ladder, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: descriptor = json.loads(context.artifacts["export_descriptor"].payload) @@ -409,6 +459,19 @@ class UKPackageInventoryKernel(KernelBase): Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, seed_source=SeedSource.NONE ) + def implementation_hash(self) -> str: + from .. import artifact_files, trace + + # The inventory records each evidence file through artifact_files + # (trace's file digest) and encodes itself canonically (microcosm#931). + return source_hash( + type(self), + artifact_files, + trace, + canonical_json, + dependencies=self.capabilities.dependencies, + ) + def run(self, context: KernelContext) -> KernelResult: report = json.loads(context.artifacts["export_readback"].payload) if ( @@ -683,6 +746,16 @@ def _require_one_reference(supporting: Mapping[str, Any]) -> None: ) +def _require_full_gate_admission(payload: bytes | Mapping, refusal: str) -> None: + """Refuse with ``refusal`` unless a full gate report permits the artifact. + + An admission read only: its caller computes nothing from the report. + """ + _, enforcement = decode_full_gate_report(payload) + if not enforcement["artifact_permitted"]: + raise ValueError(refusal) + + def _spine_gate_evidence(context: KernelContext): from ..stage_evidence import decode_stage_evidence @@ -1353,16 +1426,61 @@ class UKFullHoldoutKernel(KernelBase): ) def implementation_hash(self) -> str: + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.exact_k as calibrate_exact_k + import microcosm.calibrate.gates as calibrate_gates + import microcosm.calibrate.geography_constants as geography_constants + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.initialization as calibrate_initialization + import microcosm.calibrate.matrix as calibrate_matrix + import microcosm.calibrate.monetary_binding as monetary_binding + import microcosm.calibrate.registry as calibrate_registry + import microcosm.calibrate.solve as calibrate_solve + import microcosm.calibrate.target as calibrate_target + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import cross_grain, gate_battery, holdout, stage_evidence from . import ( + council_tax_constants, + cross_grain_declarations, dataset_size, + full_problem, + graph_population, graph_targets, + high_value_homes, + ledger_targets, local_doctrine, + local_geography, local_rowwise, + local_target_census, + local_targets, national_doctrine, + off_surface_cells, + scotland_brma_la_overlap, + size_checkpoint, + solve_progress, + support_floor, target_weights, + uprating_holds, + weighted_integrity, ) from .solve_scope import uk_solve_fingerprint + # The holdout also computes through the problem it reconstructs (the + # local surface: full_problem, ledger_targets, local_targets, + # cross_grain and its declarations, the off-surface cells, the + # uprating holds, the Scotland BRMA overlap, the council tax bands, + # high-value homes' metric names, the support floor, local_geography, + # geography_ladder, and the frame on national_frame's schema through + # graph_population), the folds and rowwise solves it runs (holdout, + # local_target_census, size_checkpoint, solve_progress, + # weighted_integrity, the engine adapter's contracts, and + # microcosm.calibrate's solver, matrix, gates, exact-K, initialisation, + # targets, registry, hierarchy, monetary checks and UK-wide geography), + # the problem it decodes (calibrate's artifacts), and its report's + # encoding (gate_battery's _json_safe, stage_evidence and the + # canonical encoder) (microcosm#931). The preflight report only + # admits it (_require_full_gate_admission). return hashlib.sha256( canonical_json( { @@ -1374,6 +1492,41 @@ def implementation_hash(self) -> str: local_doctrine, national_doctrine, target_weights, + full_problem, + ledger_targets, + local_targets, + cross_grain, + cross_grain_declarations, + off_surface_cells, + uprating_holds, + scotland_brma_la_overlap, + council_tax_constants, + high_value_homes, + support_floor, + local_geography, + geography_ladder, + graph_population, + national_frame, + holdout, + local_target_census, + size_checkpoint, + solve_progress, + weighted_integrity, + uk_engine_adapter, + calibrate_solve, + calibrate_matrix, + calibrate_gates, + calibrate_exact_k, + calibrate_initialization, + calibrate_target, + calibrate_registry, + calibrate_hierarchy, + monetary_binding, + geography_constants, + calibrate_artifacts, + gate_battery, + stage_evidence, + canonical_json, dependencies=self.capabilities.dependencies, ), # The holdout repeats the solve: it keys on the solve @@ -1393,11 +1546,10 @@ def run(self, context: KernelContext) -> KernelResult: from .graph_targets import reconstruct_uk_full_problem_inputs from .local_rowwise import rotated_uk_local_holdout - _, preflight = decode_full_gate_report(context.artifacts["preflight"].payload) - if not preflight["artifact_permitted"]: - raise ValueError( - "Rotated holdout cannot run after a blocking source preflight." - ) + _require_full_gate_admission( + context.artifacts["preflight"].payload, + "Rotated holdout cannot run after a blocking source preflight.", + ) if context.params["skip_holdout"]: report = { "report_only": True, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py index 60e8a7ba0e..897fe4db92 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py @@ -39,7 +39,42 @@ class UKInspectionExportPrepareKernel(graph_terminal.UKExportPrepareKernel): ref = "uk.full-export.prepare-inspection@1" def implementation_hash(self) -> str: - return source_hash(sys.modules[__name__], graph_terminal) + from microcosm.frame import materialize as frame_materialize + + from .. import gates + from . import ( + atomic_area_support, + geography_ladder, + graph_population, + hmrc_uprating, + local_authority_input, + national_frame, + rowwise_dataset, + rowwise_geography, + weight_year, + ) + + # The description computes through what the export kernel's does + # (microcosm#931): the ladder gate's results and region codes, the + # area code frames, the engine tables and clone index columns, the + # dataset year's weights and the engine's projection factor, the + # frame it rebuilds, and the descriptor's canonical encoding. + return source_hash( + sys.modules[__name__], + graph_terminal, + gates, + atomic_area_support, + geography_ladder, + graph_population, + hmrc_uprating, + local_authority_input, + national_frame, + rowwise_dataset, + rowwise_geography, + weight_year, + frame_materialize, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: if context.params.get("inspection_export") is not True: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 92c3902b5e..6a1a8c1495 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -3,6 +3,8 @@ # ruff: noqa: F403, F405 import functools import hashlib +from collections.abc import Mapping +from types import MappingProxyType from typing import NamedTuple from microcosm.graph import graph_schema, orrery_document @@ -885,23 +887,21 @@ def changed_module_bytes(path): # The rule behind the bindings above, for every helper (microcosm#931): # ``source_hash`` does not follow imports, so a node's hash must name every # module whose code can change what the node computes. The tests below -# derive those modules from what a stage's transform, or a dense population -# kernel, names, and follow each named definition into the module that +# derive those modules from what a stage's transform, or a UK graph kernel's +# own code, names, and follow each named definition into the module that # defines it, through re-exports and module attributes, to a fixed point. #: The packages the rule follows: every first-party package, each under the #: microcosm namespace (microcosm-build, -calibrate, -data, -diagnostics, -#: -fit, -frame and -graph). It binds what the stages compute through there -#: as in microcosm.build: microcosm.fit's QRF, model, categorical, donor and -#: held-out modules; microcosm.frame's materialize and rules modules and the -#: engine adapter, which national_frame stamps every frame through; -#: microcosm.graph's keyed uniforms and their canonical encoder; and -#: microcosm.calibrate's region tier. It stops at the core modules every node -#: runs through (``_CORE_MODULES``). The dense population kernels stay on -#: microcosm.build (``_DENSE_SCOPE``) until the UK's other graph kernels take -#: the rule. +#: -fit, -frame and -graph). It binds what the stages and kernels compute +#: through there as in microcosm.build: microcosm.fit's QRF, model, +#: categorical, donor and held-out modules; microcosm.frame's materialize and +#: rules modules and the engine adapter, which national_frame stamps every +#: frame through; microcosm.graph's keyed uniforms, canonical encoder, +#: weight-update receipt and content store; and microcosm.calibrate's solver, +#: targets and region tier. It stops at the core modules every node runs +#: through (``_CORE_MODULES``). _HELPER_SCOPE = "microcosm." -_DENSE_SCOPE = "microcosm.build." #: Where the rule stops. What a stage reads from these is its own #: declaration, which its node key already hashes as the stage contract: its #: operations, artifacts and resource pins (graph._stage_contract_sha256). @@ -967,7 +967,9 @@ def changed_module_bytes(path): #: Checks that read values only to decide whether to refuse, each with why it #: cannot change what its caller computes. The rule does not follow them, and #: a definition that names nothing else of value needs no binding for them; -#: test_the_refusal_only_helpers_return_and_write_nothing checks each one. +#: test_the_refusal_only_helpers_return_and_write_nothing checks each one. A +#: kernel's method is named ``Class.method``; the kernel rule leaves it out of +#: what the kernel runs. _REFUSAL_ONLY_HELPERS: dict[tuple[str, str], str] = { ( "microcosm.build.uk_runtime.graph_evidence", @@ -976,6 +978,20 @@ def changed_module_bytes(path): "refuses a node unless the spine gates admitted its input; the dense " "sample node calls it before it computes anything" ), + ( + "microcosm.build.uk_runtime.graph_terminal", + "_require_full_gate_admission", + ): ( + "decodes a full gate report and refuses, with its caller's message, " + "unless the report permits the artifact; the export description and " + "the rotated holdout call it before they compute anything" + ), + ("microcosm.build.uk_runtime.graph_calibration", "UKDenseSolveKernel._admit"): ( + "refuses the dense solve unless its declared admission holds: the " + "source preflight's report permits the artifact, or the bound spine's " + "provenance binds its gate report; the solve calls it before it " + "computes anything" + ), ("microcosm.build.uk_runtime.cgt_structure", "_assert_closed_world_operations"): ( "compares a stage's declared operations with the reviewed mapping, in " "order and in full, and refuses on any difference; the stages' " @@ -1015,9 +1031,9 @@ def changed_module_bytes(path): "column, or declares neither, and counts as before (checked)" ), } -#: The same for the dense population kernels, which run outside the spine -#: graph. -_DENSE_UNBOUND_NAMES: dict[tuple[str, str], str] = { +#: The same for the UK graph kernels: names a kernel reads from a module its +#: key does not hash, each with why. +_KERNEL_UNBOUND_NAMES: dict[tuple[str, str], str] = { ("microcosm.build.uk_runtime.geography_sources", name): ( "the LAD23 names lookup's pinned URL and digest: local_authority_input " "refuses an input whose declared source differs from them, and reads " @@ -1025,6 +1041,76 @@ def changed_module_bytes(path): ) for name in ("LAD23_NAMES_SHA256", "LAD23_NAMES_URL") } +#: Where the kernel rule stops, each with why a kernel that names the +#: definition need not hash what lies beyond it; +#: test_the_kernel_rule_stops_only_where_its_reasons_hold checks each. +_KERNEL_STOPS: dict[tuple[str, str], str] = { + ("microcosm.build.uk_runtime.graph_kernels", "_FixtureTransformResolver"): ( + "resolves the hermetic H2 fixture's transform for the node's stage, " + "which _fixture_implementations builds and which is the stage's own " + "transform: the stage rule holds what each transform computes " + "through, from where graph_kernels builds it" + ), + ("microcosm.build.uk_runtime.spi_spine", "uk_spi_support_mass_change_reason"): ( + "_minimal_frame rebuilds the SPI support allocation's mass record " + "through it for the hmrc_spi_income_spine stage alone, inside the " + "branch on that stage's name, and that stage's node hashes everything " + "it computes through" + ), + ("microcosm.build.uk_runtime.battery_bindings", "UK_GATE_REGISTRY"): ( + "maps each UK gate to its evaluator. A battery evaluates only the " + "gates its node's manifest declares, so how the gate batteries' keys " + "bind what each evaluator computes through is held for a ruling with " + "the held kernels (microcosm#931): the national battery's evaluators " + "too. The bound spine builds a battery run through it only for the " + "checkpoint's gate digests, which read the manifest; the run reads " + "the registry only to refuse a parameter its binding does not declare" + ), +} +#: Kernels the rule does not yet hold, each with why, all held for a ruling +#: (microcosm#931): two gate batteries whose phases share one key but declare +#: different gates, and the target kernels, whose keys microcosm#1497's +#: import record holds. +_TARGET_RECORD_REASON = ( + "test_uk_target_compile_modules holds this kernel's key to " + "microcosm#1497's import record, which lists each import of a module the " + "kernel hashes that it leaves unhashed and may only shrink. Binding what " + "this rule finds it computing through would add to the record every such " + "import of each module bound" +) +_HELD_KERNELS: dict[str, str] = { + "uk.spine-gates@1": ( + "the assembled and transferred spine phases share one key, and only " + "the transferred phase declares the gates that check the CGT asset " + "type domain and most of the stage-health checks. Binding what those " + "compute through into the shared key would re-key the assembled gate, " + "which every later spine stage waits on, as the re-review of " + "microcosm#1420 (E) kept the energy level out of it" + ), + "uk.full-gates.preflight@1": ( + "runs the calibrated battery's code, one class body and one code " + "hash, for the preflight phase only. The dense solve keys on it, so " + "binding the terminal phase's modules into the shared hash would " + "re-solve on any terminal-gate edit, which microcosm#1301 and #1306 " + "keyed out" + ), + "uk.full-gates@1": ( + "shares its code hash with the preflight, and its terminal-only " + "publication code keys it through its terminal_code parameter " + "(microcosm#1306)" + ), + **{ + ref: _TARGET_RECORD_REASON + for ref in ( + "uk.full.target_compilation@1", + "uk.full.target_selection@1", + "uk.full.measures@1", + "uk.full.problem@1", + "uk.full.national_targets@1", + "uk.full.national_problem@1", + ) + }, +} #: Method and function names that change their receiver or an argument in #: place. _IN_PLACE_CALLS = frozenset( @@ -1398,36 +1484,67 @@ def _helper_walk( modules and the listed refusal-only checks are neither followed nor returned.""" + module, name = root + node = _module_definitions(module, scope).definitions[name] + walk, _ = _walk( + {root}, + [(found, root) for found in _named(module, node, scope)], + scope, + follow_core=follow_core, + ) + return walk + + +def _walk( + roots: set[tuple[str, str]], + seeds: list[tuple[tuple[str, str], object]], + scope: str, + *, + follow_core: bool = False, + halts: Mapping[tuple[str, str], str] = MappingProxyType({}), +) -> tuple[_Walk, dict[tuple[str, str], frozenset]]: + """The walk behind ``_helper_walk`` and ``_kernel_walk``: ``roots`` are + reached as given, and each ``(definition, namer)`` seed is followed as + ``_helper_walk`` follows what its root names. A definition in ``halts`` is + neither followed nor returned; the second result maps each one met to + what named it.""" + reached: dict[str, set[str]] = {} + for module, name in roots: + reached.setdefault(module, set()).add(name) stops: set[tuple[str, str]] = set() core: set[str] = set() - pending, seen = [root], set() + met: dict[tuple[str, str], set] = {} + pending, seen = list(seeds), set(roots) while pending: - item = pending.pop() + item, namer = pending.pop() + if item in halts: + met.setdefault(item, set()).add(namer) + continue if item in seen: continue seen.add(item) module, name = item if module.startswith(_DECLARATION_MODULES): continue - if item != root: - if module in _CORE_MODULES: - core.add(module) - if not follow_core: - continue - kind = _helper_kind(module, name, scope) - if kind != "value": - if kind != "check": - stops.add(item) + if module in _CORE_MODULES: + core.add(module) + if not follow_core: continue + kind = _helper_kind(module, name, scope) + if kind != "value": + if kind != "check": + stops.add(item) + continue reached.setdefault(module, set()).add(name) node = _module_definitions(module, scope).definitions[name] - pending.extend(_named(module, node, scope)) - return _Walk( + pending.extend((found, item) for found in _named(module, node, scope)) + walk = _Walk( {module: frozenset(names) for module, names in reached.items()}, frozenset(stops), frozenset(core), ) + return walk, {item: frozenset(namers) for item, namers in met.items()} def _helper_reach( @@ -1664,8 +1781,17 @@ def test_the_core_modules_are_on_every_nodes_path() -> None: def test_the_refusal_only_helpers_return_and_write_nothing() -> None: + import ast + for module, name in _REFUSAL_ONLY_HELPERS: - node = _module_definitions(module, _HELPER_SCOPE).definitions[name] + owner, _, method = name.rpartition(".") + node = _module_definitions(module, _HELPER_SCOPE).definitions[owner or name] + if owner: + (node,) = [ + item + for item in node.body + if isinstance(item, ast.FunctionDef) and item.name == method + ] assert _writes_nothing(node), (module, name) @@ -1837,50 +1963,467 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( assert followed.core == core -def test_the_dense_population_kernels_hash_each_helper_they_compute_through() -> None: - """The same rule for the dense population kernels that share - ``_PopulationKernel``'s hash (sample, normalize, expand, locations, - geography mapping, pool and gate): every module holding code one of them - can run to compute a value is in the hash they share, and every name the - rule stops at outside it is listed in ``_DENSE_UNBOUND_NAMES`` with its - reason (microcosm#931). The rule starts at each kernel's class; the spine - gate admission check the sample kernel calls is refusal-only and is not - followed. It follows microcosm.build only (``_DENSE_SCOPE``) until the - UK's other graph kernels take the first-party rule.""" +class _KernelWalk(NamedTuple): + """What a kernel's own code can compute through (``_kernel_walk``).""" + + reach: dict[str, frozenset[str]] + stops: frozenset[tuple[str, str]] + core: frozenset[str] + met: dict[tuple[str, str], frozenset] + + +def _kernel_classes(kernel_class: type) -> list[type]: + """The first-party classes a kernel's code comes from, in its method + resolution order, without the core modules' (``KernelBase``).""" + + return [ + cls + for cls in kernel_class.__mro__ + if cls.__module__.startswith(_HELPER_SCOPE) + and cls.__module__ not in _CORE_MODULES + ] + + +def _calls_super(node) -> bool: + import ast + + return any( + isinstance(sub, ast.Call) + and isinstance(sub.func, ast.Name) + and sub.func.id == "super" + for sub in ast.walk(node) + ) + + +def _statement_names(statement) -> list[str]: + import ast + + if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef): + return [statement.name] + if isinstance(statement, ast.Assign | ast.AnnAssign): + targets = ( + statement.targets + if isinstance(statement, ast.Assign) + else [statement.target] + ) + return [ + sub.id + for target in targets + for sub in ast.walk(target) + if isinstance(sub, ast.Name) + ] + return [] + + +def _kernel_body(kernel_class: type) -> list[tuple[type, object, str]]: + """What a kernel of ``kernel_class`` runs: each method or class attribute + with its class and name, from the first class in the method resolution + order that defines it, as Python resolves it, and a base's definition too + where the one that wins calls ``super()``. ``implementation_hash`` is left + out, since it computes the key: an edit to what it calls that changes what + it returns changes the key. So are the listed refusal-only methods.""" + + chosen: dict[str, object | None] = {} + body = [] + for cls in _kernel_classes(kernel_class): + definitions = _module_definitions(cls.__module__, _HELPER_SCOPE).definitions + for statement in definitions[cls.__name__].body: + for name in _statement_names(statement): + if name in chosen: + previous = chosen[name] + if previous is not None and _calls_super(previous): + chosen[name] = statement + body.append((cls, statement, name)) + continue + if name == "implementation_hash" or ( + (cls.__module__, f"{cls.__name__}.{name}") in _REFUSAL_ONLY_HELPERS + ): + chosen[name] = None + continue + chosen[name] = statement + body.append((cls, statement, name)) + return body + + +def _kernel_walk(kernel_class: type, scope: str = _HELPER_SCOPE) -> _KernelWalk: + """What a kernel's own code can compute through: the rule followed from + what ``_kernel_body`` names, as from a stage's transform, stopping also at + ``_KERNEL_STOPS``. Its classes are in ``reach``; ``met`` maps each listed + stop it meets to what names it there (a definition, or ``Class.attribute`` + for the kernel's own code).""" + + classes = _kernel_classes(kernel_class) + seeds = [ + (found, (cls.__module__, f"{cls.__name__}.{name}")) + for cls, statement, name in _kernel_body(kernel_class) + for found in _named(cls.__module__, statement, scope) + ] + walk, met = _walk( + {(cls.__module__, cls.__name__) for cls in classes}, + seeds, + scope, + halts=_KERNEL_STOPS, + ) + return _KernelWalk(walk.reach, walk.stops, walk.core, met) + + +def _hashed_modules(kernel) -> frozenset[str]: + """Every module ``kernel.implementation_hash()`` hashes, recorded at each + ``source_hash`` it reaches, in whichever module calls it: every loaded + module's ``source_hash`` name stands in for the recorder during one call, + after a first call has imported what the hash imports.""" + import inspect + import sys + from types import ModuleType + + from microcosm.graph import kernel as graph_kernel + + original = graph_kernel.source_hash + recorded: list[object] = [] + + def record(*objects, **dependencies): + recorded.extend(objects) + return original(*objects, **dependencies) + + kernel.implementation_hash() + patched = [ + module + for module in list(sys.modules.values()) + if isinstance(module, ModuleType) + and getattr(module, "__dict__", {}).get("source_hash") is original + ] + for module in patched: + module.source_hash = record + try: + kernel.implementation_hash() + finally: + for module in patched: + module.source_hash = original + return frozenset( + (item if isinstance(item, ModuleType) else inspect.getmodule(item)).__name__ + for item in recorded + ) + + +@functools.cache +def _uk_graph_kernels() -> MappingProxyType: + """Every kernel the UK registries register, by ref: the spine's (the + hermetic registry, whose stage kernels the production spine also uses) + with its gate battery, the full build's, its gate batteries and terminal, + certification and inspection kernels, and the national line's.""" + + from microcosm.build.uk_runtime.full_certification import ( + register_uk_full_certification_kernel, + ) + from microcosm.build.uk_runtime.graph_build import register_uk_full_kernels + from microcosm.build.uk_runtime.graph_evidence import register_spine_gate_kernel + from microcosm.build.uk_runtime.graph_national import register_uk_national_kernels + from microcosm.build.uk_runtime.graph_terminal import ( + register_uk_full_gate_kernels, + register_uk_terminal_kernels, + ) + from microcosm.build.uk_runtime.inspection_export import ( + register_uk_inspection_export_kernel, + ) + + registry = uk_registry(graph=uk_spine_graph()) + register_spine_gate_kernel( + registry, + spec=load_country_spec("uk"), + engine=object(), + engine_identity="declaration-only", + ) + register_uk_full_kernels(registry) + register_uk_full_gate_kernels( + registry, coverage_engine=object(), engine_identity="declaration-only" + ) + register_uk_terminal_kernels(registry) + register_uk_full_certification_kernel(registry) + register_uk_inspection_export_kernel(registry) + register_uk_national_kernels(registry, resolver_factory=None) + return MappingProxyType({ref: registry.get(ref) for ref in registry.refs()}) + + +def test_the_kernel_rule_sees_every_kernel_the_uk_runtime_defines() -> None: + """The kernels the rule enumerates are every kernel class the UK runtime + and the shared atomic geography define, so a new one cannot escape it, + and the held kernels are among them (microcosm#931).""" + import ast + import importlib + + from microcosm.build import graph_atomic_geography + from microcosm.build.uk_runtime import graph_kernels + files = sorted(Path(graph_kernels.__file__).parent.glob("*.py")) + files.append(Path(graph_atomic_geography.__file__)) + defined = set() + for path in files: + module = ( + graph_atomic_geography.__name__ + if path.stem == "graph_atomic_geography" + else f"{graph_kernels.__package__}.{path.stem}" + ) + for node in ast.parse(path.read_text(encoding="utf-8")).body: + if not isinstance(node, ast.ClassDef): + continue + declares_ref = any( + "ref" in _statement_names(statement) for statement in node.body + ) or any( + isinstance(sub, ast.Attribute) + and sub.attr == "ref" + and isinstance(sub.ctx, ast.Store) + for sub in ast.walk(node) + ) + if declares_ref: + defined.add(getattr(importlib.import_module(module), node.name)) + kernels = _uk_graph_kernels() + assert {type(kernel) for kernel in kernels.values()} == defined + assert set(_HELD_KERNELS) <= set(kernels) + + +def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: + """The stage rule, for every other UK graph kernel (microcosm#931): each + module holding code a kernel's own methods can run to compute a value is + in what its implementation hash really hashes, recorded at the + ``source_hash`` calls it makes, and every name the rule stops at outside + that hash is listed in ``_KERNEL_UNBOUND_NAMES`` with its reason. + + The rule starts at what the kernel runs (``_kernel_body``: its methods by + method resolution order, without the hash itself and the listed + refusal-only methods), not at its class, and stops at the declaration, + refusal-only and core-module tables as for the stages, and at + ``_KERNEL_STOPS``. Where a class has several kernels (the spine's stage + kernels, one per stage), each one's key is held to it. The held kernels + (``_HELD_KERNELS``) are not. + + What the rule cannot see, beside what the stage rule cannot: a name a + package exports lazily, through its module ``__getattr__``. The one a + kernel calls is microcosm.graph's ``load_source``, through which the + create kernel loads the H2 fixture's tables when no root transform is + bound; the production spine always binds one.""" from microcosm.build.uk_runtime import graph_population, rowwise_geography - kernels = [ - member - for _, member in inspect.getmembers(graph_population, inspect.isclass) - if member.__module__ == graph_population.__name__ - and issubclass(member, graph_population._PopulationKernel) - and member is not graph_population._PopulationKernel - and member.implementation_hash + kernels = _uk_graph_kernels() + by_class: dict[type, list[tuple[str, object]]] = {} + for ref, kernel in sorted(kernels.items()): + by_class.setdefault(type(kernel), []).append((ref, kernel)) + unhashed: dict[str, list[str]] = {} + unlisted: set[tuple[str, str]] = set() + met: set[tuple[str, str]] = set() + for kernel_class, members in by_class.items(): + held = {ref in _HELD_KERNELS for ref, _ in members} + assert len(held) == 1, kernel_class + if held == {True}: + continue + walk = _kernel_walk(kernel_class) + met |= set(walk.met) + for ref, kernel in members: + hashed = _hashed_modules(kernel) + missing = sorted(set(walk.reach) - hashed) + if missing: + unhashed[ref] = missing + unlisted.update(item for item in walk.stops if item[0] not in hashed) + assert not unhashed, unhashed + assert unlisted == set(_KERNEL_UNBOUND_NAMES), sorted( + unlisted ^ set(_KERNEL_UNBOUND_NAMES) + ) + assert met == set(_KERNEL_STOPS), sorted(met ^ set(_KERNEL_STOPS)) + # The dense population kernels that share _PopulationKernel's hash + # (sample, normalize, expand, locations, geography mapping, pool and + # gate) mint their clones through rowwise_geography's helpers. + shared = [ + kernel + for kernel in kernels.values() + if type(kernel).implementation_hash is graph_population._PopulationKernel.implementation_hash ] - assert {kernel.ref for kernel in kernels} >= { + assert {kernel.ref for kernel in shared} >= { "uk.full.sample@1", "uk.full.expand@1", "uk.full.geography_gate@1", } - unhashed = {} - unlisted = set() - for kernel in kernels: - hashed = hashed_module_names(kernel.__new__(kernel), graph_population) - # The dense clones are minted by rowwise_geography's helpers. - assert rowwise_geography.__name__ in hashed - walk = _helper_walk((graph_population.__name__, kernel.__name__), _DENSE_SCOPE) - missing = sorted(set(walk.reach) - hashed) - if missing: - unhashed[kernel.ref] = missing - unlisted.update(item for item in walk.stops if item[0] not in hashed) - assert not unhashed, unhashed - assert unlisted == set(_DENSE_UNBOUND_NAMES), sorted( - unlisted ^ set(_DENSE_UNBOUND_NAMES) + for kernel in shared: + assert rowwise_geography.__name__ in _hashed_modules(kernel), kernel.ref + + +def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: + """Each listed kernel stop is met only where its reason says + (microcosm#931).""" + import ast + + from microcosm.build.uk_runtime import ( + battery_bindings, + graph_kernels, + release_certification, + spi_spine, ) + kernels = _uk_graph_kernels() + namers: dict[tuple[str, str], dict[str, frozenset]] = {} + for ref, kernel in kernels.items(): + if ref in _HELD_KERNELS: + continue + for stop, by in _kernel_walk(type(kernel)).met.items(): + namers.setdefault(stop, {})[ref] = by + + # The fixture resolver resolves the stage's own transform: of what holds a + # value, it names only the builder the stage rule starts from. + resolver = (graph_kernels.__name__, "_FixtureTransformResolver") + definitions = _module_definitions(graph_kernels.__name__, _HELPER_SCOPE).definitions + named = { + item + for item in _named( + graph_kernels.__name__, definitions[resolver[1]], _HELPER_SCOPE + ) + if not item[0].startswith(_DECLARATION_MODULES) + and item[0] not in _CORE_MODULES + and _helper_kind(*item, _HELPER_SCOPE) == "value" + } + assert named == {(graph_kernels.__name__, "_fixture_implementations")} + assert set(namers[resolver]) == { + kernel.ref + for kernel in kernels.values() + if isinstance( + kernel, graph_kernels.UKStageKernel | graph_kernels.UKExpandStageKernel + ) + } + + # The SPI support allocation's mass record: _minimal_frame names it only + # inside its branch on the SPI income stage, whose node hashes all it + # computes through. + stage = "hmrc_spi_income_spine" + reason = (spi_spine.__name__, "uk_spi_support_mass_change_reason") + assert {namer for by in namers[reason].values() for namer in by} == { + (graph_kernels.__name__, "_minimal_frame") + } + minimal = definitions["_minimal_frame"] + guarded = set() + for branch in ast.walk(minimal): + if ( + isinstance(branch, ast.If) + and ast.unparse(branch.test) == f"context.params.get('stage') == '{stage}'" + ): + guarded |= {id(sub) for item in branch.body for sub in ast.walk(item)} + references = [ + sub + for sub in ast.walk(minimal) + if isinstance(sub, ast.Name | ast.Attribute) + and reason in _named(graph_kernels.__name__, sub, _HELPER_SCOPE) + ] + assert references and all(id(sub) in guarded for sub in references) + reach = _helper_walk(reason) + hashed = _hashed_modules(kernels[f"uk.stage.{stage}@1"]) + assert set(reach.reach) | {module for module, _ in reach.stops} <= hashed + + # The gate registry: the national battery evaluates through it (held); + # the bound spine derives its gate digests through it, in + # _manifest_digests, which returns the battery run's digests alone. + registry = (battery_bindings.__name__, "UK_GATE_REGISTRY") + assert namers[registry] == { + "uk.full.bound_spine@1": frozenset( + {(release_certification.__name__, "_manifest_digests")} + ), + "uk.full.national-gates@1": frozenset( + {("microcosm.build.uk_runtime.graph_national", "UKNationalGateKernel.run")} + ), + } + digests = _module_definitions( + release_certification.__name__, _HELPER_SCOPE + ).definitions["_manifest_digests"] + (returned,) = [sub for sub in ast.walk(digests) if isinstance(sub, ast.Return)] + assert {ast.unparse(value) for value in returned.value.values} == { + "run.gates_manifest_sha256", + "run.policy_sha256", + "run.spec_fingerprint", + } + + +def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> None: + """Control: the kernel rule follows what a kernel's methods name, + through its bases by method resolution order (a base method it inherits, + and one it overrides only where it calls ``super()``), and leaves out its + implementation hash, a listed refusal-only method, a base method it + overrides without ``super()`` and what lies behind a listed kernel stop, + which it reports with what named it. The recorder sees each module the + hash passes to ``source_hash``, in whichever module calls it.""" + import importlib + import uuid + + scratch = f"kernel_rule_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / scratch + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + sources = { + "computed": "def computed(x):\n return x + 1\n", + "inherited": "def base_only(x):\n return x * 2\n", + "assisted": "def kept(x):\n return x * 3\n", + "overridden": "def replaced(x):\n return x - 2\n", + "refused": "def check(x):\n return x < 0\n", + "keyonly": "def salt():\n return 'salt'\n", + "behind": "def beyond(x):\n return x * 5\n", + "table": "from .behind import beyond\n\nTABLE = {'a': beyond}\n", + "keyed": ( + "from microcosm.graph import source_hash\n\n" + "from . import computed, keyonly\n\n" + "def key():\n return source_hash(computed, keyonly)\n" + ), + "base": ( + "from .assisted import kept\n" + "from .inherited import base_only\n" + "from .overridden import replaced\n\n" + "class Base:\n" + " def run(self, x):\n" + " return base_only(x)\n\n" + " def prepare(self, x):\n" + " return replaced(x)\n\n" + " def assist(self, x):\n" + " return kept(x)\n" + ), + "kernel": ( + "from .base import Base\n" + "from .computed import computed\n" + "from .keyed import key\n" + "from .keyonly import salt\n" + "from .refused import check\n" + "from .table import TABLE\n\n" + "class Kernel(Base):\n" + " def implementation_hash(self):\n" + " return salt() + key()\n\n" + " def _admit(self, x):\n" + " if check(x):\n" + " raise ValueError('refused')\n\n" + " def prepare(self, x):\n" + " return x\n\n" + " def run(self, x):\n" + " self._admit(x)\n" + " return computed(x) + super().run(x) + TABLE['a'](x)\n" + ), + } + for module, source in sources.items(): + (package / f"{module}.py").write_text(source) + prefix = f"microcosm.{scratch}." + monkeypatch.setitem( + _REFUSAL_ONLY_HELPERS, (f"{prefix}kernel", "Kernel._admit"), "control" + ) + monkeypatch.setitem(_KERNEL_STOPS, (f"{prefix}table", "TABLE"), "control") + kernel_class = importlib.import_module(f"{prefix}kernel").Kernel + walk = _kernel_walk(kernel_class, scope=prefix) + assert { + module.removeprefix(prefix): names for module, names in walk.reach.items() + } == { + "kernel": {"Kernel"}, + "base": {"Base"}, + "computed": {"computed"}, + "inherited": {"base_only"}, + "assisted": {"kept"}, + } + assert walk.met == { + (f"{prefix}table", "TABLE"): frozenset({(f"{prefix}kernel", "Kernel.run")}) + } + assert _hashed_modules(kernel_class()) == {f"{prefix}computed", f"{prefix}keyonly"} + #: Adapters the spine build installs around a stage's own transform, each with #: why it computes no value. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index faa142ec88..ece2d8dc3b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -223,6 +223,29 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "parameter; no kernel's run reads it" ), ), + # graph_national imports these only inside the national gate and + # readback kernels' implementation hashes, which bind them there + # (microcosm#931); the national target and problem kernels never run + # those methods. + **{ + module: HashInput( + importers=("microcosm.build.uk_runtime.graph_national",), + reason=( + "imported only in uk.full.national-gates@1's or " + "uk.full.national-readback@1's implementation hash, to " + "hash it there" + ), + ) + for module in ( + "microcosm.build.gates", + "microcosm.build.trace", + "microcosm.build.uk_runtime.donor_uprating", + "microcosm.build.uk_runtime.etb_services", + "microcosm.calibrate.monetary_binding", + "microcosm.frame.adapters.policyengine_uk", + "microcosm.frame.materialize", + ) + }, } ) From 760738355c2688f6e63b1cd1ecab7c7511937784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:58:56 +0000 Subject: [PATCH 033/108] Follow up the review of the 9 Nov correction, count-check and binding commits (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three reviews of c9b486279, 753e89dc6, c19776091 and 0a9d20833 found no must. This commit takes the shoulds and nits that need no ruling. The rest go to María with the review batch: stage 42's one-copy check and a copy_rule clause for it, the fallback that can undo that check, the K = 1 hold's narrowings, the engine adapter's imports, the nine held kernels with the national gates' evaluators, and regional_home_values' seed source. Rule B (stages 42 and 26): - Tests pin what no test held. Tied whole households take the correction in row order: the earlier of two at one clone, bit for bit as before; clone 0 at three clones; and the earliest of forty tied in a deterministic scramble of eighty, more ties than an unstable sort keeps in order. The walk half of the one-copy check passes over a household an ulp below its band's support mass, and the identity receipt stays green; with the rerun answering as the stage's walk did, it reports a missing family. A one-copy household that ties the heaviest unselected one from an earlier row takes the correction, the one case where K = 1 weights move although today's candidate could carry it. Stage 26 tries exactly its unselected positive households, so a selected root that could carry the correction never does. - The docstrings and the changelog say the shared fallback can also land on a whole household, a one-copy one the check refused among them, and the rerun can then divide a different set, as under the rule before. They give the order condition (a one-copy household comes first where it outweighs the heaviest unselected one or ties it from an earlier row), the case where nothing moves, the guarded worst case (64 walk reruns, about 26 s on a synthetic vector the size of a K = 15 spine, which a national frame does not reach) and the key chaining from stage 26 on. Identity tool (the K = 1 hold): - A derived test reads the hold's premises from the spine graph and the stages' code. After the split, only the incidence clone, the #970 anchor and the residential split create, expand or reweigh rows. The split, the clone and the residential split each correct their total once, through spi_support's one-row helpers. The anchor corrects none and caps its factor at one, and every later stage passes its input's household weights on. - Tests pin the equal-halves threshold (three members with equal halves do not vouch for a family) and move the root, as well as a copy, in the folded-above-a-multiple test. - The changelog's What moves, the receipt's tolerance_policy and the receipts test docstring name the one-float tier's blind spot. Where fewer than four members show their pair untouched, a family split consistently into one copy more on the multiple can go unreported: for 6,393 of the 9,995 counts n from 4 to 9,998, against 31 where the family is held exactly. Bindings: - The create-frame normalisation and the owned-cell cast move unchanged from graph_kernels into a new graph_frames module. The bound spine hashes it in place of graph_kernels, and the spine's create, claim and stage kernels hash it too, so no national-line kernel hashes graph_kernels now. - The size search and refit hash their five extra modules in their own implementation hashes, through a shared _calibration_hash, which the kernel rule leaves out as it leaves out every implementation hash. The dense solve and the other calibration kernels no longer carry them, so an edit to local_doctrine or target_weights re-keys no national-line node; the national targets and problem hash the other three. - test_uk_graph: the national gates' hold at UK_GATE_REGISTRY moves from the stop table into _HELD_KERNEL_STOPS. A test checks that each held kernel and held stop still guards a module its hash leaves out. Another holds the executor's and key modules' definitions that kernels outside the hold name to a listed table, each with a reason: dtype_for_token, ArtifactType, ArtifactOutput, StructuralDelta and KernelRole. The Frame's reason says H2 cannot catch a Frame edit, and a test pins Weights.total as NumPy's sum. The fixture resolver's stop names the injected engine as its exception. A comment by the SeedSource.NONE assertion records that regional_home_values draws keyed uniforms, held for a ruling. - The helper-bindings changelog: the intro, "or be a listed adapter", the claim kernel's graph_frames, the NZ transport graph's geography nodes, that held kernels can replay stale outputs, and the national line's re-keys. What moves: node keys only. No value, declaration or generated file moves; cgt_support.py changes only in comments and docstrings, and the tool only in its receipt's tolerance_policy text. Of the 99 UK kernels, 64 implementation hashes move: every spine stage kernel, the create, identity and claim kernels, the bound spine, and the seven calibration and size kernels. Every spine node and every full-build node re-key, and so does the national line from its bound spine root. Neither the country nor the solve fingerprint moves. source_stages.json equals the typed projection, and the coverage manifest is current. The H2 fixture holds stage contract hashes and no declaration changed, so it is not regenerated. Tests (engine-free, synthetic): the four touched files pass (242 tests). So do 21 related files (572 tests): those that load the tool, scan uk_runtime's modules, hold the import records, or run the create, claim, bound-spine and calibration kernels. Ruff is clean on the edited files, and ci_test_plan verify is ok. Mutations under scratch, each caught: - reversed ties, and an unstable sort (by the forty-tie frame); - the walk rerun stubbed to agree; - stage 26 trying selected roots; - unselected households tried before one-copy ones (only the one-copy test fails); - the hold where members differ (by the moved root), or on one or three equal halves; - for the premises test: four corrected rows, a second correction in the clone, one in the anchor, a rescaled pass-through, and the anchor's cap at 1.5; - Weights.total as fsum; - a kernel naming graph.keys' frame_key; - the size kernels dropping target_weights; - the dense solve calling local_doctrine; - the bound spine dropping graph_frames; - a hold that guards nothing. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 4 +- changelog.d/931-uk-helper-bindings.changed.md | 24 +- .../microcosm/build/uk_runtime/cgt_support.py | 49 +-- .../microcosm/build/uk_runtime/graph_build.py | 15 +- .../build/uk_runtime/graph_calibration.py | 111 ++++--- .../build/uk_runtime/graph_frames.py | 53 ++++ .../build/uk_runtime/graph_kernels.py | 44 +-- .../engine_free/uk/test_uk_cgt_support.py | 150 +++++++++- .../tests/engine_free/uk/test_uk_graph.py | 213 +++++++++++-- .../uk/test_uk_high_value_home_support.py | 59 ++++ .../uk/test_uk_identity_stability_receipts.py | 281 +++++++++++++++++- tools/verify_uk_identity_stability.py | 10 +- 12 files changed, 851 insertions(+), 162 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/graph_frames.py diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index 326ceef804..bfead53b14 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -2,5 +2,5 @@ The CGT support split (`cgt_support_split`) counts each selected household's cop - **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where an exact-total correction fell on a member, and at K > 1 by more than the snap absorbs. So the count check does not recount the fold. At K = 1 a family of more than three members whose clone pair sums are all one weight c is held exactly: its stored count n must be the count of a weight w with fl(w / n) = c. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so such a family has a member no correction moved, whose pair sum only the later stages' roundings can have moved off w / n. Where at least four members show their two clone halves at equal weights, one of those is uncorrected, and equal halves mean the #970 anchor and the residential arms left its pair sum at exactly w / n, so c is exactly w / n. Elsewhere the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member. Every other family, and every family at K > 1, keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. That bracket opens for a family whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a family the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to at most the multiple itself, or the multiple and the float either side of it where the hold reaches one float. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. - **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception. Both stages' notes say where the correction lands. The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. -- **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy carries it only where its corrected weight leaves the walk's selection and its one copy as they were, so the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the identity tool's recompute would have found a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy. The row is then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate wherever no selected household left at one copy weighs as much, which needs every unselected household to weigh at most 60 / K. The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. -- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them. The same can happen at stage 42 where a selected household left at one copy weighs as much as every unselected one. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the K = 1 hold and the count bracket. Against the exact recount it replaces, at K = 1 it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, but no longer one of three or fewer members, or one whose members fold to different weights. At K > 1 no such family is reported. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. +- **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy carries it only where its corrected weight leaves the walk's selection and its one copy as they were, so wherever one of the 64 carries it, the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the identity tool's recompute would have found a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy, then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. The fallback does not consult that check, so its row can also be a whole household, among them a one-copy household the check passed over; the rerun can then divide a different set. That can happen on synthetic frames whose one-copy household sits one to three ulps below its band's support mass, as it could under the rule before, and the identity tool then reports a missing family. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate unless a selected household left at one copy outweighs it, or ties it from an earlier row, which needs every unselected household to weigh at most 60 / K (to within the K-ulp snap at K > 1). The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. A selected household left at one copy that can carry it also costs a rerun of the selection walk, about 0.41 s at that size, so 64 that the check passes over would take about 26 s; a national frame does not reach that, since its 64 heaviest whole households are unselected ones, far heavier than 60 / K. +- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot and a later one of the 64 can, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them; where none of the 64 can, nothing moves. At stage 42, and at K = 1 too, a selected household left at one copy that outweighs the heaviest unselected household, or ties it from an earlier row, comes first, which needs every unselected household to be as light; where it can carry the correction and keep the split, it takes it even where the heaviest unselected household could have, so those two rows' weights move by rounding amounts. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the K = 1 hold and the count bracket. Against the exact recount it replaces, at K = 1 it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, where its members fold to one weight and at least four of them show their clone pairs untouched, for all but 31 of the 9,995 counts n from 4 to 9,998. Where fewer do, the hold reaches one float, which takes in the multiple itself for 6,393 of those counts, so such a family goes unreported there, as the full-chain test's 900 family does. It no longer reports one of three or fewer members, or one whose members fold to different weights. At K > 1 no such family is reported. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`, and node keys chain on their inputs' keys, so every spine node from stage 26 on, and every node downstream of the spine, re-keys with them. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 2897a84d4a..356400639c 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,4 +1,4 @@ -Every UK spine stage now hashes each first-party module that its transform computes through, and every other UK graph kernel, but nine held for a ruling, each one its own code computes through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. +Every UK spine stage now hashes each first-party module its transform computes through, and every other UK graph kernel, except nine held for a ruling, hashes each one its own code computes through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. - **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports and module attributes. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: - six listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels); @@ -6,7 +6,7 @@ Every UK spine stage now hashes each first-party module that its transform compu - the listed core modules every node runs through (see Scope); - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; two such names are listed. - A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies it to the UK's other graph kernels (see Kernels). Two more cover the transforms the production spine build installs itself; a transform installed inside the source adapter or the rung sampler must come from its stage's module, since neither hands the node what the wrapped transform declares. Control tests pin each way of naming a helper that the rule follows, each kind it skips, the core-module stop, and each way a refusal-only check could write. + A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies it to the UK's other graph kernels (see Kernels). Two more cover the transforms the production spine build installs itself; a transform installed inside the source adapter or the rung sampler must come from its stage's module or be a listed adapter, since neither hands the node what the wrapped transform declares. Control tests pin each way of naming a helper that the rule follows, each kind it skips, the core-module stop, and each way a refusal-only check could write. - **Scope.** The rule follows every first-party package: `microcosm-build`, `-calibrate`, `-data`, `-diagnostics`, `-fit`, `-frame` and `-graph`, all under the `microcosm.` namespace. What the stages compute through outside `microcosm.build` is now bound as it is inside it, 71 bindings in 47 stages: - `microcosm.fit.qrf` and `fit.model` in 11 stages: the nine that fit through the regime-gated QRF, and `spi_employment_shell` and `aps_occupation`, whose classifier resolves its fit weights through `fit.model` and whose held-out scores and folds use the QRF's; - `microcosm.frame.materialize`, whose `engine_tables` copies a frame's tables, in the three stages of `spi_spine` and the band donors; @@ -16,7 +16,7 @@ Every UK spine stage now hashes each first-party module that its transform compu - `microcosm.calibrate.geography_constants`, the UK region tier, in `hmrc_cgt_gains_spine`, `isa_subscriptions` and `spi_housing_shell`. Listed instead, each with a reason a test checks: - - the core modules every node runs through, where the rule stops, since binding one would re-key every node on any edit to it. They are the Frame's three modules (`microcosm.frame.bundle`, `schema` and `weights`), which a stage's transform meets all three or none of; the graph executor's (`microcosm.graph.executor` and `population`); and the modules that compute node keys (`keys`, `decl` and `kernel`), which no stage's transform meets. No spine key hashes them. An edit to the Frame that changes values is left to its own tests and to H1, whose fixtures run it through `run_graph` against stored bytes. One to the executor's modules that changes a value on H2's fixture fails H2, which compares the graph's spine with the legacy stage plan's. One that changes what `node_key` or `source_hash` returns changes every key. A test checks that the walk from `run_graph` meets each of them. `microcosm.graph.canonical` also encodes every key, but `regional_home_values` draws through it, so it is bound there instead; + - the core modules every node runs through, where the rule stops, since binding one would re-key every node on any edit to it. They are the Frame's three modules (`microcosm.frame.bundle`, `schema` and `weights`), which a stage's transform meets all three or none of; the graph executor's (`microcosm.graph.executor` and `population`); and the modules that compute node keys (`keys`, `decl` and `kernel`), which no stage's transform meets. No spine key hashes them. H2 cannot catch an edit to the Frame, since the graph's spine and the legacy stage plan both run through it, and H1, whose fixtures run four wrapped kernels through `run_graph` against stored bytes, catches one only on those kernels' frames; an edit that changes a UK value is otherwise left to the Frame's own tests, and a test pins that `Weights.total`, which the UK exact-total corrections target, is NumPy's sum of the weights. One to the executor's modules that changes a value on H2's fixture fails H2, which compares the graph's spine with the legacy stage plan's. One that changes what `node_key` or `source_hash` returns changes every key. A test checks that the walk from `run_graph` meets each of them. `microcosm.graph.canonical` also encodes every key, but `regional_home_values` draws through it, so it is bound there instead. That stage draws through `microcosm.graph.randomness`, whose contract asks for `SeedSource.KEYED`, while every spine stage kernel declares `NONE`; which it should declare is held for a ruling; - `microcosm.frame.rules`' `assert_rules_engine_country`, the fourth refusal-only check, which every stage that takes an engine calls first; - `SOURCE_HOUSEHOLD_ID_COLUMN` from `microcosm.frame.concepts`, CopyLineage's default source column, the second listed name. The support channel writes the column by its literal name. Every stage that reads the constant declares the column by that name, which its key hashes, and counts before the copies only beside a copy column, or declares neither, so a renamed constant leaves the lineage readers to refuse or count as before. @@ -35,20 +35,20 @@ Every UK spine stage now hashes each first-party module that its transform compu The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input`, `gates` and `spi_support`, whose replacement strata the sample reads. - **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. Through `graph_implementation_dependencies`, which the root node hashes, it declares `national_sampling` at every rung, since it resets the root's indexes at f100 too, and `frame_sampling` only below f100, where it samples. Its behaviour is unchanged. - **Kernels.** A second test applies the rule to every kernel the UK registries register but the nine held below: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; and the national gates and readback. It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at three listed kernel stops, each with a reason a test checks: - - the hermetic fixture's transform resolver, since the stage rule holds each stage's transform; + - the hermetic fixture's transform resolver, since the stage rule holds each stage's transform, but for the engine it injects into the engine stages, whose own imports are outside the rule (see Scope); - `_minimal_frame`'s rebuild of the SPI support allocation's mass record, which only `hmrc_spi_income_spine` takes, and whose reach that stage's node hashes; - - `UK_GATE_REGISTRY` (see Held for a ruling). + - `UK_GATE_REGISTRY`, which the bound spine reaches only for its checkpoint's gate digests. The national gates evaluate through it and are held there, in a table of their own (see Held for a ruling). - The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: + The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. Another holds the executor's and key modules' definitions that the kernels' own code names to a listed table, each with a reason: the dtype tokens' map, which the executor casts every node's owned cells through anyway, two artifact declaration records and two capability enums. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: - the seven dense population kernels' shared hash: the engine adapter, the canonical encoder and the weight-update receipt; - the geography identity: `household_lineage`, `source_lineage`, `microcosm.frame.concepts` and the canonical encoder; - property value: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `tenure_constants`, the keyed uniforms and the canonical encoder; - property band: `tenure_constants`; BRMA and RFDR: `receipt_counts`; - private rent: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `national_frame`, `receipt_counts`, the keyed uniforms and the canonical encoder; - - the spine's identity and claim kernels: `national_frame`; + - the spine's identity and claim kernels: `national_frame`, and the claim kernel `graph_frames` too, through which it casts the cells it claims (see the bound spine); - the four atomic geography kernels' shared hash: the content store, which `load_source_bytes` takes, and `frame.materialize`; - - the seven calibration and size kernels' shared hash: `national_frame`, `calibrate.hierarchy`, `calibrate.monetary_binding` and the canonical encoder, and, for the size search and refit, `target_weights`, `local_doctrine`, `local_targets`, `national_doctrine` and `calibrate.geography_constants`; - - the bound spine: `frame.materialize`, the engine adapter, `content_identity`, `hmrc_uprating`, `gate_battery`, `gates`, `trace`, `graph_kernels` (its create-frame normalisation) and the canonical encoder; + - the seven calibration and size kernels' shared hash: `national_frame`, `calibrate.hierarchy`, `calibrate.monetary_binding` and the canonical encoder. The size search and refit hash `target_weights`, `local_doctrine`, `local_targets`, `national_doctrine` and `calibrate.geography_constants` too, in their own implementation hashes, since the dense solve and the other kernels sharing that hash do not compute through them; + - the bound spine: `frame.materialize`, the engine adapter, `content_identity`, `hmrc_uprating`, `gate_battery`, `gates`, `trace`, `graph_frames` and the canonical encoder. The create-frame normalisation it shares with the spine's root, and the owned-cell cast the spine's kernels share, moved unchanged from `graph_kernels` into the new `graph_frames`, which the spine's create, claim and stage kernels hash, so an edit to the spine's kernel module no longer re-keys the national line; - the export description: `gates`, `atomic_area_support`, `graph_population`, `hmrc_uprating`, `local_authority_input`, `rowwise_dataset`, `rowwise_geography`, `frame.materialize` and the canonical encoder. The inspection export binds the same modules, and `geography_ladder`, `national_frame` and `weight_year` too; - the export readback: `artifact_files`, `trace`, `geography_ladder`, the engine adapter, `frame.materialize` and the canonical encoder; the package inventory: `artifact_files`, `trace` and the canonical encoder; - the certification: `artifact_files`, `trace`, `gate_battery`, `gates`, `graph_evidence`, `calibration_run` and the canonical encoder; @@ -57,7 +57,7 @@ Every UK spine stage now hashes each first-party module that its transform compu - the national readback: `artifact_files`, `trace`, `frame.materialize`, the engine adapter, `geography_ladder`, `graph_population`, `hmrc_uprating` and the canonical encoder. The LAD23 names' pin stays listed, now for every kernel that reads it. `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside the national gates' and readback's hashes. -- **Held for a ruling.** Nine kernels stay outside the rule, each listed with its reason: - - the spine gates, and the full build's preflight and calibrated gates. Each battery shares one key across phases that declare different gates, which it evaluates through `UK_GATE_REGISTRY`, where the rule stops for every kernel. Binding what one phase's evaluators reach would re-key the assembled spine gate, or the preflight and with it the dense solve, on edits that cannot change them. The national gates' evaluators are held with them; +- **Held for a ruling.** Nine kernels stay outside the rule, each listed with its reason. Until the ruling, a held kernel can replay a stale output after an edit to a module it computes through but does not hash; a test checks that every held kernel's walk still meets such a module, so a hold cannot outlive its need: + - the spine gates, and the full build's preflight and calibrated gates. Each battery shares one key across phases that declare different gates, which it evaluates through `UK_GATE_REGISTRY`. Binding what one phase's evaluators reach would re-key the assembled spine gate, or the preflight and with it the dense solve, on edits that cannot change them. The national gates are held with them at the registry alone: the rule holds everything else they compute through, and the same test checks that the walk past the registry still meets a module their hash leaves out; - the target compilation, selection, measures and problem kernels, and the national targets and problem. microcosm#1497's import record holds their keys and may only shrink. Binding what the rule finds would add to it every unhashed import of each module bound. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport registry's geography nodes. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Only the size search and refit hash their five modules, and the national line runs neither, so none of its nodes re-keys on an edit to `local_doctrine` or `target_weights` either; its targets and problem kernels compute through the other three and hash them. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index cec8b32178..6b8f06796a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -125,8 +125,14 @@ #: a support split left whole, heaviest first, before spi_support's shared #: fallback (microcosm#931; :func:`_exact_support_weights`, which the #: high-value home support split shares). Each candidate that cannot carry the -#: correction costs a few dozen sums of the frame's weights, so the cap bounds -#: the search on a national frame. +#: correction costs a few dozen sums of the frame's weights. At the CGT split +#: a selected household left at one copy that can carry it also costs a rerun +#: of the selection walk (``keeps_the_split``), so 64 such candidates that the +#: check passes over would cost 64 walks, about 26 s at the 0.41 s one walk +#: takes on a synthetic vector the size of a ``K = 15`` spine. A national +#: frame does not reach that, since its 64 heaviest whole households are +#: unselected ones, far heavier than ``60 / K``, so the cap bounds the search +#: there to the sums. CGT_SUPPORT_CORRECTION_CANDIDATES = 64 #: The one exception to "every copy carries weight / copies" that the #: ``copy_rule`` of both support splits declares, built on the cap so the @@ -604,12 +610,15 @@ def split_cgt_support_households( :data:`CGT_SUPPORT_CORRECTION_CANDIDATES` heaviest that can carry it, so every root and copy weighs bitwise ``w / n``. A selected one carries it only where its corrected weight leaves the walk's selection and its one - copy as they were, so the rule rerun on the output divides the same - households. Only where none of them can does spi_support's shared - fallback place it, and that row can be a root or copy, then off ``w / - n`` by about one ulp of the total (:func:`_exact_support_weights`). - Refuses a frame the incidence clone stage (``cgt_incidence_clone``) has - already seen or one this stage has already split. + copy as they were, so wherever one of those candidates carries it, the + rule rerun on the output divides the same households. Only where none of + them can does spi_support's shared fallback place it, unchanged and + without that check. Its row can then be a root or copy, off ``w / n`` by + about one ulp of the total, or a whole household, among them a one-copy + one the check passed over, after which the rule rerun on the output can + divide a different set (:func:`_exact_support_weights`). Refuses a frame + the incidence clone stage (``cgt_incidence_clone``) has already seen or + one this stage has already split. """ validate_uk_national_frame(frame) @@ -710,13 +719,13 @@ def split_cgt_support_households( def keeps_the_split(row: int, weight: float) -> bool: """Whether whole household ``row`` at ``weight`` leaves the split as it was. - An unselected household sits past its band's crossing, so its weight - moves neither the walk nor its count of one. A selected household - left at one copy can move both, so it carries the correction only - where the walk rerun on its corrected weight selects the same - households and that weight still counts one copy: the rule rerun on - the stage's output then divides the same households the same way, as - the identity tool's recompute does. + An unselected household sits past its band's crossing, or in no + walked band, so its weight moves neither the walk nor its count of + one. A selected household left at one copy can move both, so it + carries the correction only where the walk rerun on its corrected + weight selects the same households and that weight still counts one + copy: the rule rerun on the stage's output then divides the same + households the same way, as the identity tool's recompute does. """ if not selected[row]: @@ -840,10 +849,12 @@ def _exact_support_weights( spi_support's shared multi-candidate correction place it, unchanged: its candidates (the heaviest, last, first, lightest and middle positive rows) can be a root or copy of a divided household, which is then off ``w / n`` - by about one ulp of the total. On the CGT nesting scenario at 7 and 13 - clones, for one, every whole household sits in the half of NumPy's - pairwise sum whose other half carries exactly half an ulp of the total, - so no change to its weight rounds the total onto the target. + by about one ulp of the total, or a whole household, a one-copy household + that check refused among them, since it does not consult + ``keeps_the_split``. On the CGT nesting scenario at 7 and 13 clones, for + one, every whole household sits in the half of NumPy's pairwise sum whose + other half carries exactly half an ulp of the total, so no change to its + weight rounds the total onto the target. """ rows = np.asarray(whole, dtype=np.int64) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py index c7b9fe4ba3..56cc3949e4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -45,7 +45,8 @@ uk_calibration_nodes, ) from .graph_evidence import SPINE_BUILD_STATE_TYPE -from .graph_kernels import UKClaimKernel, UKIdentityKernel, _normalize_create_frame +from .graph_frames import normalize_create_frame +from .graph_kernels import UKClaimKernel, UKIdentityKernel from .graph_population import ( UKPoolCheckpointKernel, append_uk_population_nodes, @@ -392,14 +393,16 @@ def implementation_hash(self) -> str: from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from .. import gate_battery, gates, trace - from . import content_identity, graph_kernels, hmrc_uprating + from . import content_identity, graph_frames, hmrc_uprating # The node also computes through the H5 it loads (frame_materialize, # the engine adapter's contracts), the checkpoint's content identity # and pinned engine (content_identity, hmrc_uprating), the gate # digests it derives through a battery run (gate_battery, gates, - # trace), graph_kernels' create-frame normalisation and the canonical - # encoding of its artifacts (microcosm#931). + # trace), the create-frame normalisation it shares with the spine's + # root (graph_frames, not the spine's kernel module, so an edit there + # does not re-key the national line) and the canonical encoding of its + # artifacts (microcosm#931). return source_hash( type(self), national_frame, @@ -412,7 +415,7 @@ def implementation_hash(self) -> str: gate_battery, gates, trace, - graph_kernels, + graph_frames, canonical_json, ) @@ -432,7 +435,7 @@ def run(self, context: KernelContext) -> KernelResult: provenance = calibration_run.strict_spine_provenance_from_sidecar( sidecar_path, sidecar, gate_report_path=context.sources["uk_spine_gates"] ) - created = _normalize_create_frame(frame, context) + created = normalize_create_frame(frame, context) return KernelResult( frame=created, artifacts={ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py index 7bf9d83c49..7387afa408 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py @@ -265,53 +265,68 @@ def restore_uk_graph_result( return replace(result, frame=final_frame) -class _CalibrationKernel(KernelBase): - def implementation_hash(self): - import microcosm.calibrate.geography_constants as geography_constants - import microcosm.calibrate.hierarchy as calibrate_hierarchy - import microcosm.calibrate.monetary_binding as monetary_binding - - from . import ( - local_doctrine, - local_targets, - national_doctrine, - national_frame, - target_weights, +def _size_solve_modules() -> tuple[object, ...]: + """What the size search and refit compute through beyond the shared hash: + the target weights, with the doctrine, local target areas and UK-wide + geography their solves read (microcosm#931).""" + + import microcosm.calibrate.geography_constants as geography_constants + + from . import local_doctrine, local_targets, national_doctrine, target_weights + + return ( + target_weights, + local_doctrine, + local_targets, + national_doctrine, + geography_constants, + ) + + +def _calibration_hash(kernel: KernelBase, *modules: object) -> str: + """The implementation hash every solve kernel shares, over ``modules`` too. + + Every solve kernel computes through the frame it rebuilds on + national_frame's schema, the targets' hierarchy and monetary checks in + microcosm.calibrate and the canonical encoding of its artifacts. The size + search and refit add what :func:`_size_solve_modules` names, which the + dense solve and the other kernels sharing this hash do not compute + through (microcosm#931). test_uk_graph derives each kernel's set. + """ + + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.monetary_binding as monetary_binding + + from . import national_frame + + return hashlib.sha256( + canonical_json( + { + "solver": CalibrateAdamKernel().implementation_hash(), + "adapter": source_hash( + type(kernel), + dataset_size, + size_checkpoint, + calibration_artifacts, + context_frame, + select_exact_k, + HardConcrete, + contribution_initialization, + national_frame, + calibrate_hierarchy, + monetary_binding, + canonical_json, + *modules, + dependencies=kernel.capabilities.dependencies, + ), + } ) + ).hexdigest() - # Every solve kernel shares this hash. They also compute through the - # frame they rebuild on national_frame's schema, the targets' - # hierarchy and monetary checks in microcosm.calibrate, the canonical - # encoding of their artifacts, and the size search's and refit's - # target weights with the doctrine, local target areas and UK-wide - # geography they read (microcosm#931); test_uk_graph derives the set. - return hashlib.sha256( - canonical_json( - { - "solver": CalibrateAdamKernel().implementation_hash(), - "adapter": source_hash( - type(self), - dataset_size, - size_checkpoint, - calibration_artifacts, - context_frame, - select_exact_k, - HardConcrete, - contribution_initialization, - national_frame, - calibrate_hierarchy, - monetary_binding, - canonical_json, - target_weights, - local_doctrine, - local_targets, - national_doctrine, - geography_constants, - dependencies=self.capabilities.dependencies, - ), - } - ) - ).hexdigest() + +class _CalibrationKernel(KernelBase): + def implementation_hash(self): + return _calibration_hash(self) class UKSizeCheckpointImportKernel(_CalibrationKernel): @@ -591,6 +606,9 @@ class UKSizeSearchKernel(_ObservedSolveKernel): dependencies=_DEPENDENCIES, ) + def implementation_hash(self): + return _calibration_hash(self, *_size_solve_modules()) + def run(self, context): frame, problem = _inputs(context) dense = _dense(context, frame, problem) @@ -704,6 +722,9 @@ class UKSizeRefitKernel(_ObservedSolveKernel): dependencies=_DEPENDENCIES, ) + def implementation_hash(self): + return _calibration_hash(self, *_size_solve_modules()) + def run(self, context): frame, problem = _inputs(context) dense = _dense(context, frame, problem) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_frames.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_frames.py new file mode 100644 index 0000000000..9f97750b08 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_frames.py @@ -0,0 +1,53 @@ +"""The owned cells and created frames the UK graph kernels hand the executor. + +A kernel returns each cell its node owns at the dtype the node declares for it, +and a root kernel returns its created frame with those cells cast and the build +period stamped. The spine's create, claim and stage kernels and the national +line's bound spine root share these helpers, so each hashes this module rather +than the module of another kernel family (microcosm#931). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pandas as pd + +from microcosm.frame import Frame +from microcosm.graph.population import dtype_for_token + +if TYPE_CHECKING: + from microcosm.graph import KernelContext + +__all__ = ["cast_owned", "normalize_create_frame"] + + +def cast_owned(series: pd.Series, dtype: str) -> pd.Series: + """``series`` at the dtype its node declares for it (the token ``dtype``).""" + + target = dtype_for_token(dtype) + if series.dtype != target: + series = series.astype(target) + return series + + +def normalize_create_frame(frame: Frame, context: KernelContext) -> Frame: + """A created frame with its node's owned cells at their declared dtypes + and the build period of the node's ``time_period`` parameter.""" + + tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + for owned in context.node.outputs: + tables[owned.entity][owned.column] = cast_owned( + tables[owned.entity][owned.column], owned.dtype + ).array + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata={ + **frame.metadata, + "time_period": str(context.params.get("time_period", "2024")), + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 96e8835234..3618ec2985 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -96,6 +96,7 @@ frs_spine, frs_take_up, gambling_participation, + graph_frames, hmrc_capital_gains, hmrc_company_car_statistics, hmrc_income, @@ -131,6 +132,7 @@ uc_relationships, was_wealth, ) +from .graph_frames import cast_owned, normalize_create_frame from .national_frame import UK_NATIONAL_SCHEMA if TYPE_CHECKING: @@ -918,6 +920,8 @@ def _implementation_hash(kernel: object, stage: str, transform: object | None) - # the time period of their frames through it (microcosm#931). It stamps # each frame it builds with the engine adapter's ISA income contract and # savings income projection, so every kernel hashes the adapter with it. + # Every kernel casts the cells it owns, and the root normalises the frame + # it creates, through graph_frames. dependencies = getattr(transform, "graph_implementation_dependencies", None) return source_hash( type(kernel), @@ -925,6 +929,7 @@ def _implementation_hash(kernel: object, stage: str, transform: object | None) - graph_evidence, national_frame, uk_engine_adapter, + graph_frames, _stage_artifacts, _mass_log_payload, _stage_module(stage), @@ -1659,39 +1664,13 @@ def _id_index(frame: Frame, entity: str) -> pd.Index: return pd.Index(frame.table(entity)[id_column].to_numpy(copy=True), name=id_column) -def _cast_owned(series: pd.Series, dtype: str) -> pd.Series: - target = dtype_for_token(dtype) - if series.dtype != target: - series = series.astype(target) - return series - - def _owned_series(frame: Frame, entity: str, column: str, dtype: str) -> pd.Series: - values = _cast_owned(frame.table(entity)[column].copy(deep=True), dtype) + values = cast_owned(frame.table(entity)[column].copy(deep=True), dtype) values.index = _id_index(frame, entity) values.name = column return values -def _normalize_create_frame(frame: Frame, context: KernelContext) -> Frame: - tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} - for owned in context.node.outputs: - tables[owned.entity][owned.column] = _cast_owned( - tables[owned.entity][owned.column], owned.dtype - ).array - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata={ - **frame.metadata, - "time_period": str(context.params.get("time_period", "2024")), - }, - ) - - class UKCreateKernel(KernelBase): """Load fixture CSV tables or invoke the bound FRS root transform.""" @@ -1725,7 +1704,7 @@ def run(self, context: KernelContext) -> KernelResult: f"The UK root transform returned {type(frame).__name__}, not Frame." ) return KernelResult( - frame=_normalize_create_frame(frame, context), + frame=normalize_create_frame(frame, context), artifacts=_stage_artifacts( "frs_spine", self.transform, @@ -1764,15 +1743,18 @@ class UKClaimKernel(KernelBase): def implementation_hash(self) -> str: # The claimed cells are indexed by national_frame's schema's id - # columns (microcosm#931). + # columns and cast through graph_frames (microcosm#931). return source_hash( - type(self), national_frame, dependencies=self.capabilities.dependencies + type(self), + national_frame, + graph_frames, + dependencies=self.capabilities.dependencies, ) def run(self, context: KernelContext) -> KernelResult: columns = { (owned.entity, owned.column): pd.Series( - _cast_owned( + cast_owned( context.tables[owned.entity][owned.column].copy(deep=True), owned.dtype, ).array, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index eb32dcd795..a729036684 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -1126,12 +1126,18 @@ def test_split_records_pool_exhaustion_and_empty_bands_without_refusing(): @dataclass(frozen=True) class _CorrectionCall: - """What the split handed the exact-total correction, and its selection.""" + """What the split handed the exact-total correction, and its selection. + + ``reselected`` holds each rerun of the walk the correction made, one per + selected household left at one copy that could carry it at a weight that + still counts one copy. + """ values: np.ndarray target: float whole: np.ndarray selected: np.ndarray + reselected: tuple[np.ndarray, ...] = () def _recorded_correction( @@ -1167,7 +1173,9 @@ def recording_select(*args, **kwargs): ((values, target, whole),) = calls # The first selection is the stage's; any later one reruns the walk for # a selected household left at one copy that could carry the correction. - return result, _CorrectionCall(values, target, whole, selections[0]) + return result, _CorrectionCall( + values, target, whole, selections[0], reselected=tuple(selections[1:]) + ) def _moved_rows(result, call: _CorrectionCall) -> list[int]: @@ -1292,6 +1300,144 @@ def test_a_household_the_correction_would_divide_is_passed_over(monkeypatch): assert cgt_support_copy_counts([weights[3]], clone_count=1).tolist() == [1] +def test_a_household_whose_correction_would_move_the_walk_is_passed_over( + monkeypatch, +): + # Both households are selected: the walk takes the first, an ulp below + # its band's support mass of 40 (2 x 1.25 x 16), and crosses on the + # second. The first is left at one copy, the one household the split + # leaves whole, and the correction it could carry still counts one copy, + # but it lifts the household onto 40, where the walk rerun on its + # corrected weight stops at it and leaves the second unselected. So it is + # passed over (microcosm#931), and the shared fallback corrects the + # divided root instead. + below = float(np.nextafter(40.0, -np.inf)) + frame = support_frame( + weights=[below, 82.4], wealth=[9.0, 8.0], incomes=[BAND_INCOMES[0]] * 2 + ) + result, call = _recorded_correction(monkeypatch, frame, {0: {250_000: 16.0}}) + + assert call.selected.tolist() == [True, True] + assert call.whole.tolist() == [0] + corrected = _exact_total_correction( + call.values, target=call.target, correction_index=0 + ) + assert corrected is not None and corrected[0] >= 40.0 + assert cgt_support_copy_counts([corrected[0]], clone_count=1).tolist() == [1] + (reselected,) = call.reselected + assert reselected.tolist() == [True, False] + assert _moved_rows(result, call) == [1] + weights = np.asarray(result.frame.weights_for("household").values) + assert weights[0] == below + assert weights[1] == pytest.approx(41.2, rel=1e-12) + + +def test_a_one_copy_household_tied_with_the_heaviest_unselected_takes_it_first( + monkeypatch, +): + # Where the rule before's one candidate, the heaviest unselected + # household, carries the correction, the weights are the ones it gave, + # but for one case (microcosm#931): a selected household left at one + # copy comes first where it outweighs that household or, as here, ties + # it from an earlier row, which needs every unselected household to + # weigh at most 60. The walk takes the 145.5 household (three copies) and + # crosses on the first 37.87 one, left at one copy. The unselected 37.87 + # household could carry the correction, but the one-copy household + # comes first and carries it, keeping the split, so the weights differ + # from the rule before's on those two rows, and every divided row stays + # bitwise w / 3. + frame = support_frame( + weights=[145.5, 37.87, 7.05, 26.65, 32.73, 37.87], + wealth=[9e6, 8e6, 4e3, 3e3, 2e3, 1.0], + incomes=[BAND_INCOMES[0]] * 6, + ) + result, call = _recorded_correction(monkeypatch, frame, {0: {250_000: 65.77}}) + + assert call.selected.tolist() == [True, True, False, False, False, False] + assert call.whole.tolist() == [1, 2, 3, 4, 5] + assert call.values[1] == call.values[5] + before, carried = _correction_before(call) + assert carried + assert np.flatnonzero(before != call.values).tolist() == [5] + assert _moved_rows(result, call) == [1] + (reselected,) = call.reselected + assert reselected.tolist() == call.selected.tolist() + weights = np.asarray(result.frame.weights_for("household").values) + assert np.flatnonzero(weights != before).tolist() == [1, 5] + assert weights[1] == pytest.approx(37.87, rel=1e-12) + household = result.frame.table("household") + assert household[CGT_SUPPORT_COPIES_COLUMN].tolist() == [3, 1, 1, 1, 1, 1, 3, 3] + assert weights[[0, 6, 7]].tolist() == [145.5 / 3] * 3 + + +def test_tied_whole_households_take_the_correction_in_row_order(monkeypatch): + # Tied weights are tried in row order (a stable sort), as np.argmax took + # the first of them before (microcosm#931). At one clone the earlier of + # two unselected households tied at the heaviest weight carries the + # correction, though the later could too, and the weights are the rule + # before's bit for bit. At three clones a source household's clones tie, + # and clone 0, the first, carries it. + top = {0: {250_000: 1_000.0}, 37_700: {250_000: 0.0}} + frame = support_frame( + weights=[274.0, 54.34, 42.59, 54.34], + wealth=[9e6, 3e3, 2e3, 1e3], + incomes=[BAND_INCOMES[0]] + [BAND_INCOMES[37_700]] * 3, + ) + result, call = _recorded_correction(monkeypatch, frame, top) + + assert call.whole.tolist() == [1, 2, 3] + assert not call.selected[call.whole].any() + assert call.values[1] == call.values[3] + assert _carries(call, 1) and _carries(call, 3) + later = _exact_total_correction(call.values, target=call.target, correction_index=3) + before, carried = _correction_before(call) + assert carried and before.tobytes() != later.tobytes() + assert _moved_rows(result, call) == [1] + weights = np.asarray(result.frame.weights_for("household").values) + assert weights.tobytes() == before.tobytes() + + cloned = clone_first_frame( + support_frame( + weights=[273.0, 82.77, 25.07], + wealth=[9e6, 2e3, 1e3], + incomes=[BAND_INCOMES[0]] + [BAND_INCOMES[37_700]] * 2, + ), + 3, + ) + result, call = _recorded_correction(monkeypatch, cloned, top, clone_count=3) + + household = result.frame.table("household") + sources = household["source_household_id"].to_numpy()[: len(call.values)] + clones = np.flatnonzero(sources == 2) + assert clones.tolist() == [1, 4, 7] + assert all(_carries(call, row) for row in clones.tolist()) + assert _moved_rows(result, call) == [1] + assert household["household_clone_index"].iloc[1] == 0 + before, carried = _correction_before(call) + weights = np.asarray(result.frame.weights_for("household").values) + assert carried and weights.tobytes() == before.tobytes() + + # At fifteen clones every source household's clones tie fifteen ways. + # Here forty of eighty unselected households tie, in a fixed golden-ratio + # scramble, more than an unstable sort keeps in row order, and the + # earliest tied row still carries the correction. + keys = ((np.arange(80) + 1) * 0.6180339887) % 1.0 + position = np.argsort(np.argsort(keys, kind="stable"), kind="stable") + frame = support_frame( + weights=[274.0, *np.where(position < 40, 54.34, position - 38.5).tolist()], + wealth=[9e6, *np.arange(1_000.0, 920.0, -1.0).tolist()], + incomes=[BAND_INCOMES[0]] + [BAND_INCOMES[37_700]] * 80, + ) + result, call = _recorded_correction(monkeypatch, frame, top) + + tied = call.whole[call.values[call.whole] == call.values[call.whole].max()] + assert len(tied) == 40 and all(_carries(call, row) for row in tied.tolist()) + assert _moved_rows(result, call) == [int(tied.min())] == [2] + before, carried = _correction_before(call) + weights = np.asarray(result.frame.weights_for("household").values) + assert carried and weights.tobytes() == before.tobytes() + + def test_split_counts_suppressed_cells_and_reports_channels_when_present(): result = split_cgt_support_households( support_frame( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 6a1a8c1495..0ceb3d04d5 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -914,9 +914,13 @@ def changed_module_bytes(path): _FRAME_REASON = ( "the spine stages' transforms compute on it, every spine stage kernel " "builds one, and the executor holds every population version in one. No " - "spine key hashes it, so an edit that changes values is left to its own " - "tests and to H1, whose fixtures run it through run_graph against stored " - "bytes" + "spine key hashes it, and H2 cannot catch an edit to it, since the " + "graph's spine and the legacy stage plan both run through it. H1, whose " + "fixtures run four wrapped kernels through run_graph against stored " + "bytes, catches one only on those kernels' frames, so an edit that changes " + "a UK value is otherwise left to the Frame's own tests; the totals the UK " + "exact-total corrections target are pinned here " + "(test_the_frame_totals_the_uk_corrections_target_are_numpys_sums)" ) #: The core modules every node runs through, which the rule neither follows #: nor asks a stage to bind, each with what runs through it and what catches @@ -964,6 +968,46 @@ def changed_module_bytes(path): "what it reads changes its key" ), } +#: The definitions of the executor's and the key modules that the UK graph +#: kernels' own code names, each with why naming it binds nothing the kernel's +#: key misses (test_the_core_definitions_the_kernels_name_are_listed). The +#: Frame's definitions, which kernels build and read frames with, are the +#: Frame's reason's; a stage's transform meets the Frame's modules alone +#: (test_the_core_modules_are_on_every_nodes_path). +_KERNEL_CORE_NAMES: dict[tuple[str, str], str] = { + ("microcosm.graph.population", "dtype_for_token"): ( + "maps a node's declared dtype token to the dtype its owned cells are " + "stored at. The executor stores every node's owned cells at the dtype " + "it returns (patch, checked), so a kernel that casts its cells through " + "it first hands the executor what the executor stores anyway, and an " + "edit to it moves every node's stored dtypes alike: H2 catches one on " + "the dtypes its spine fixture carries, and graph.population's own " + "tests the rest" + ), + ("microcosm.graph.decl", "ArtifactType"): ( + "a frozen record of an artifact type's name and version, which a " + "kernel names to declare or check its artifacts' types; node_key " + "hashes each node's declared artifact types as part of its normative " + "declaration" + ), + ("microcosm.graph.decl", "ArtifactOutput"): ( + "a frozen record pairing an artifact's name with its type, which the " + "atomic geography gate names to declare and check its outputs; " + "node_key hashes each node's declared artifact outputs" + ), + ("microcosm.graph.decl", "StructuralDelta"): ( + "the enum a kernel's capabilities declare its row-set change in; the " + "executor refuses a kernel whose declared change differs from its " + "node's, and node_key binds the declared value with the rest of the " + "kernel's capabilities" + ), + ("microcosm.graph.kernel", "KernelRole"): ( + "the enum a kernel's capabilities declare its role in, by which the " + "executor holds a gate's outcome to the gate outcomes and derives a " + "release's tier from its gate ancestors; node_key binds the declared " + "value with the rest of the kernel's capabilities" + ), +} #: Checks that read values only to decide whether to refuse, each with why it #: cannot change what its caller computes. The rule does not follow them, and #: a definition that names nothing else of value needs no binding for them; @@ -1049,7 +1093,11 @@ def changed_module_bytes(path): "resolves the hermetic H2 fixture's transform for the node's stage, " "which _fixture_implementations builds and which is the stage's own " "transform: the stage rule holds what each transform computes " - "through, from where graph_kernels builds it" + "through, from where graph_kernels builds it. The one exception is " + "the PolicyEngineUKEngine it builds and injects into the engine " + "stages: they bind the three engine modules, but no rule follows the " + "adapter's own first-party imports, which is held for a ruling " + "(microcosm#931)" ), ("microcosm.build.uk_runtime.spi_spine", "uk_spi_support_mass_change_reason"): ( "_minimal_frame rebuilds the SPI support allocation's mass record " @@ -1058,19 +1106,35 @@ def changed_module_bytes(path): "it computes through" ), ("microcosm.build.uk_runtime.battery_bindings", "UK_GATE_REGISTRY"): ( - "maps each UK gate to its evaluator. A battery evaluates only the " - "gates its node's manifest declares, so how the gate batteries' keys " - "bind what each evaluator computes through is held for a ruling with " - "the held kernels (microcosm#931): the national battery's evaluators " - "too. The bound spine builds a battery run through it only for the " - "checkpoint's gate digests, which read the manifest; the run reads " - "the registry only to refuse a parameter its binding does not declare" + "maps each UK gate to its evaluator. The bound spine builds a battery " + "run through it only for the checkpoint's gate digests, which read the " + "manifest, and the run reads the registry only to refuse a parameter " + "its binding does not declare. A kernel that evaluates gates through " + "it is held (_HELD_KERNELS, _HELD_KERNEL_STOPS)" + ), +} +#: Kernels the rule holds at one listed stop only, for a ruling +#: (microcosm#931), each with why. Past the stop the rule holds them as any +#: other kernel. test_every_hold_still_guards_a_module_its_kernel_leaves_out +#: checks that the walk past each stop still meets a module the kernel's hash +#: leaves out. +_HELD_KERNEL_STOPS: dict[tuple[str, tuple[str, str]], str] = { + ( + "uk.full.national-gates@1", + ("microcosm.build.uk_runtime.battery_bindings", "UK_GATE_REGISTRY"), + ): ( + "evaluates the gates its manifest declares through the registry, and " + "how a gate kernel's key binds what each evaluator computes through is " + "held with the gate batteries (_HELD_KERNELS)" ), } #: Kernels the rule does not yet hold, each with why, all held for a ruling -#: (microcosm#931): two gate batteries whose phases share one key but declare +#: (microcosm#931): the gate batteries, whose phases share one key but declare #: different gates, and the target kernels, whose keys microcosm#1497's -#: import record holds. +#: import record holds. Until the ruling, a held kernel can replay a stale +#: output after an edit to a module it computes through but does not hash; +#: test_every_hold_still_guards_a_module_its_kernel_leaves_out checks that +#: each still does, so a hold cannot outlive its need. _TARGET_RECORD_REASON = ( "test_uk_target_compile_modules holds this kernel's key to " "microcosm#1497's import record, which lists each import of a module the " @@ -1486,7 +1550,7 @@ def _helper_walk( module, name = root node = _module_definitions(module, scope).definitions[name] - walk, _ = _walk( + walk, _, _ = _walk( {root}, [(found, root) for found in _named(module, node, scope)], scope, @@ -1507,13 +1571,14 @@ def _walk( reached as given, and each ``(definition, namer)`` seed is followed as ``_helper_walk`` follows what its root names. A definition in ``halts`` is neither followed nor returned; the second result maps each one met to - what named it.""" + what named it, and the third holds every core module's definition met.""" reached: dict[str, set[str]] = {} for module, name in roots: reached.setdefault(module, set()).add(name) stops: set[tuple[str, str]] = set() core: set[str] = set() + core_names: set[tuple[str, str]] = set() met: dict[tuple[str, str], set] = {} pending, seen = list(seeds), set(roots) while pending: @@ -1529,6 +1594,7 @@ def _walk( continue if module in _CORE_MODULES: core.add(module) + core_names.add(item) if not follow_core: continue kind = _helper_kind(module, name, scope) @@ -1544,7 +1610,11 @@ def _walk( frozenset(stops), frozenset(core), ) - return walk, {item: frozenset(namers) for item, namers in met.items()} + return ( + walk, + {item: frozenset(namers) for item, namers in met.items()}, + frozenset(core_names), + ) def _helper_reach( @@ -1777,6 +1847,13 @@ def test_the_core_modules_are_on_every_nodes_path() -> None: registry = uk_registry(graph=graph) for stage, (node_id, _, _) in walks.items(): kernel = registry.get(graph.node(node_id).kernel) + # regional_home_values draws its keyed uniforms through + # microcosm.graph.randomness, whose contract asks a kernel drawing + # through it to declare SeedSource.KEYED. It declares NONE, as every + # spine stage kernel does through the one stage kernel class: its + # stream is a stage parameter, which its contract hash covers, and it + # reads no generator the executor seeds. Whether it should declare + # KEYED is held for a ruling (microcosm#931 review). assert kernel.capabilities.seed_source is SeedSource.NONE, stage @@ -1970,6 +2047,7 @@ class _KernelWalk(NamedTuple): stops: frozenset[tuple[str, str]] core: frozenset[str] met: dict[tuple[str, str], frozenset] + core_names: frozenset[tuple[str, str]] def _kernel_classes(kernel_class: type) -> list[type]: @@ -2045,12 +2123,18 @@ def _kernel_body(kernel_class: type) -> list[tuple[type, object, str]]: return body -def _kernel_walk(kernel_class: type, scope: str = _HELPER_SCOPE) -> _KernelWalk: +def _kernel_walk( + kernel_class: type, + scope: str = _HELPER_SCOPE, + *, + halts: Mapping[tuple[str, str], str] | None = None, +) -> _KernelWalk: """What a kernel's own code can compute through: the rule followed from what ``_kernel_body`` names, as from a stage's transform, stopping also at - ``_KERNEL_STOPS``. Its classes are in ``reach``; ``met`` maps each listed - stop it meets to what names it there (a definition, or ``Class.attribute`` - for the kernel's own code).""" + ``halts`` (``_KERNEL_STOPS`` unless given). Its classes are in ``reach``; + ``met`` maps each listed stop it meets to what names it there (a + definition, or ``Class.attribute`` for the kernel's own code), and + ``core_names`` holds the core modules' definitions it names.""" classes = _kernel_classes(kernel_class) seeds = [ @@ -2058,13 +2142,13 @@ def _kernel_walk(kernel_class: type, scope: str = _HELPER_SCOPE) -> _KernelWalk: for cls, statement, name in _kernel_body(kernel_class) for found in _named(cls.__module__, statement, scope) ] - walk, met = _walk( + walk, met, core_names = _walk( {(cls.__module__, cls.__name__) for cls in classes}, seeds, scope, - halts=_KERNEL_STOPS, + halts=_KERNEL_STOPS if halts is None else halts, ) - return _KernelWalk(walk.reach, walk.stops, walk.core, met) + return _KernelWalk(walk.reach, walk.stops, walk.core, met, core_names) def _hashed_modules(kernel) -> frozenset[str]: @@ -2316,18 +2400,22 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: hashed = _hashed_modules(kernels[f"uk.stage.{stage}@1"]) assert set(reach.reach) | {module for module, _ in reach.stops} <= hashed - # The gate registry: the national battery evaluates through it (held); - # the bound spine derives its gate digests through it, in - # _manifest_digests, which returns the battery run's digests alone. + # The gate registry: the bound spine derives its gate digests through it, + # in _manifest_digests, which returns the battery run's digests alone; + # the national battery evaluates through it and is held there. registry = (battery_bindings.__name__, "UK_GATE_REGISTRY") - assert namers[registry] == { + held_there = {ref for ref, stop in _HELD_KERNEL_STOPS if stop == registry} + assert held_there == {"uk.full.national-gates@1"} + assert { + ref: by for ref, by in namers[registry].items() if ref not in held_there + } == { "uk.full.bound_spine@1": frozenset( {(release_certification.__name__, "_manifest_digests")} ), - "uk.full.national-gates@1": frozenset( - {("microcosm.build.uk_runtime.graph_national", "UKNationalGateKernel.run")} - ), } + assert namers[registry]["uk.full.national-gates@1"] == frozenset( + {("microcosm.build.uk_runtime.graph_national", "UKNationalGateKernel.run")} + ) digests = _module_definitions( release_certification.__name__, _HELPER_SCOPE ).definitions["_manifest_digests"] @@ -2339,6 +2427,71 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: } +def test_every_hold_still_guards_a_module_its_kernel_leaves_out() -> None: + """A hold lasts only while it is needed (microcosm#931 review): each held + kernel's walk still meets a module its hash leaves out, and so does the + walk past each held stop, so a hold that guards nothing fails here and + leaves its table. Until the ruling, such a kernel can replay a stale + output after an edit to one of those modules.""" + kernels = _uk_graph_kernels() + assert {ref for ref, _ in _HELD_KERNEL_STOPS} <= set(kernels) + assert not {ref for ref, _ in _HELD_KERNEL_STOPS} & set(_HELD_KERNELS) + unneeded = [] + for ref in _HELD_KERNELS: + kernel = kernels[ref] + if not set(_kernel_walk(type(kernel)).reach) - _hashed_modules(kernel): + unneeded.append(ref) + for ref, stop in _HELD_KERNEL_STOPS: + assert stop in _KERNEL_STOPS, (ref, stop) + kernel = kernels[ref] + past = _kernel_walk( + type(kernel), + halts={item: why for item, why in _KERNEL_STOPS.items() if item != stop}, + ) + if not set(past.reach) - _hashed_modules(kernel): + unneeded.append((ref, stop)) + assert not unneeded, unneeded + + +def test_the_core_definitions_the_kernels_name_are_listed() -> None: + """The kernel version of the stage rule's core check (microcosm#931 + review): the definitions of the core modules other than the Frame's that + the UK graph kernels' own code names, outside the held kernels, are + exactly ``_KERNEL_CORE_NAMES``, and the executor stores every node's + owned cells through the dtype tokens' map, as that entry's reason says.""" + kernels = _uk_graph_kernels() + named: set[tuple[str, str]] = set() + for ref, kernel in kernels.items(): + if ref not in _HELD_KERNELS: + named |= { + item + for item in _kernel_walk(type(kernel)).core_names + if not item[0].startswith("microcosm.frame.") + } + assert named == set(_KERNEL_CORE_NAMES), sorted(named ^ set(_KERNEL_CORE_NAMES)) + patching = _helper_walk(("microcosm.graph.population", "patch"), follow_core=True) + assert "dtype_for_token" in patching.reach["microcosm.graph.population"] + + +def test_the_frame_totals_the_uk_corrections_target_are_numpys_sums() -> None: + """H2 cannot catch an edit to the Frame, which the graph's spine and the + legacy stage plan both run through (``_FRAME_REASON``). The UK exact-total + corrections correct a vector until NumPy's sum of it equals a frame's + ``Weights.total``, so this pins that the total is that same sum, bit for + bit, on vectors whose pairwise and correctly rounded sums differ + (microcosm#931 review).""" + import math + + rng = np.random.default_rng(11) + differing = 0 + for size in (7, 129, 1_000, 78_188): + values = rng.lognormal(3.0, 2.0, size) + total = Weights(values, WeightKind.IMPORTANCE).total + assert total == float(values.sum()) + differing += total != math.fsum(values) + assert differing + + def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> None: """Control: the kernel rule follows what a kernel's methods name, through its bases by method resolution order (a base method it inherits, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py index 848493e6e8..3150fe8980 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py @@ -313,6 +313,7 @@ def carries(row: int) -> bool: return corrected is not None assert result.households_selected == 10 and len(values) == 129 + assert whole.tolist() == _unselected_positive_rows(result, values) heaviest_first = whole[np.lexsort((whole, -values[whole]))] assert heaviest_first[:64].tolist() == [*range(62, -1, -1), 64] assert float(values.sum()) != target @@ -334,6 +335,64 @@ def carries(row: int) -> bool: assert family.tolist() == [weight / 6] * 6 +def _unselected_positive_rows(result, values) -> list[int]: + """The incumbent rows the stage left whole: unselected, positive weight.""" + + household = result.frame.table("household") + copies = household[hv.HIGH_VALUE_SUPPORT_COPIES_COLUMN].to_numpy() + incumbent = ~household[HIGH_VALUE_SUPPORT_COPY_COLUMN].to_numpy(dtype=bool) + return np.flatnonzero(incumbent & (copies == 1) & (values > 0.0)).tolist() + + +def test_a_selected_root_never_carries_the_correction(monkeypatch) -> None: + # The stage's whole households are its unselected positive ones + # (microcosm#931): every household it selects becomes six rows at w / 6, + # roots included. Here the two most valuable owners are selected, and + # the root of the 379.53 one, at 379.53 / 6, outweighs both unselected + # households and could carry the correction, but only they are tried: + # the heaviest, the 36.21 one, carries it, and every root and copy stays + # bitwise w / 6. + from microcosm.build.uk_runtime.spi_support import _exact_total_correction + + weights_in = [29.45, 36.21, 379.53, 37.78] + base = _frame(len(weights_in)) + household = base.table("household").copy() + household["tenure_type"] = pd.array(["OWNED_OUTRIGHT"] * 4, dtype="string") + household["main_residence_value"] = [100_000.0 * (i + 1) for i in range(4)] + frame = uk_national_frame( + person=base.table("person"), + benunit=base.table("benunit"), + household=household, + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=np.asarray(weights_in), + ) + calls = [] + correct = hv._exact_support_weights + + def recording(values, *, target, whole, **keywords): + calls.append((np.array(values), target, np.array(whole))) + return correct(values, target=target, whole=whole, **keywords) + + monkeypatch.setattr(hv, "_exact_support_weights", recording) + result = hv.split_high_value_support_households(frame, stratum_lower=0.5, copies=6) + ((values, target, whole),) = calls + + assert result.households_selected == 2 + assert whole.tolist() == _unselected_positive_rows(result, values) == [0, 1] + assert float(values.sum()) != target + assert values[2] > max(values[0], values[1]) + assert ( + _exact_total_correction(values, target=target, correction_index=2) is not None + ) + weights = np.asarray(result.frame.weights_for("household").values) + assert result.frame.weights_for("household").total == target + assert np.flatnonzero(weights != values).tolist() == [1] + sources = result.frame.table("household")["household_id"].to_numpy() % 10**12 + for source, weight in ((3, 379.53), (4, 37.78)): + assert weights[sources == source].tolist() == [weight / 6] * 6 + + def test_the_copy_rule_names_the_one_exception() -> None: import json from pathlib import Path diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index a997b0c6d9..432b0dcab4 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1318,9 +1318,11 @@ class TestSupportSplitRecomputeAcrossGeographicClones: clone a family of more than three members whose clone pair sums are all one weight ``c`` is held exactly: the stored ``n`` must be the count of a weight that divides by ``n`` to ``c``, or to within one float of it where - fewer than four members show their clone pair untouched. Every other - family, and every family at ``K > 1``, is held to the family weights' - relative tolerance of 1e-9. That bracket opens for a family whose + fewer than four members show their clone pair untouched, a reach that can + take in the multiple of 60 itself, where a family split consistently into + one copy more goes unreported. Every other family, and every family at + ``K > 1``, is held to the family weights' relative tolerance of 1e-9. + That bracket opens for a family whose source-scaled weight ``K x w`` lies within a relative 1e-9 of a multiple of 60 (at ``K = 1``, its own weight), where it cannot tell ``n`` copies from ``n + 1``: a stored count moved on its own is left to the flags and @@ -1651,21 +1653,23 @@ def test_through_the_anchor_and_the_residential_split_the_hold_stays_sound( assert problems == {"support_split_copies_stored": 3} assert (exact, within) == (4, 2) + @pytest.mark.parametrize("member", ["root", "copy"]) def test_at_one_clone_a_family_folded_just_above_a_multiple_is_green( - self, + self, member: str ) -> None: - # One copy of household 1 (900: 15 copies of 60) carries a rounding - # amount more, as where an exact-total correction falls on a member, - # so the family folds to just above 900, where an exact recount gives - # 16. Its members no longer carry one weight, so the family is held to - # the bracket at one clone too (microcosm#931 review), which accepts - # the stage's 15, and the receipt is green; its exact recount still - # records the extra copy. + # The root or one copy of household 1 (900: 15 copies of 60) carries a + # rounding amount more, as where an exact-total correction falls on a + # member, so the family folds to just above 900, where an exact + # recount gives 16. Its members no longer carry one weight, so the + # family is held to the bracket at one clone too (microcosm#931 + # review), which accepts the stage's 15, and the receipt is green; its + # exact recount still records the extra copy. Held exactly on the + # root's weight, the moved root would report the family. split, frame = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) ids = frame.table("household")["household_id"].to_numpy() weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() (copy,) = copy_id_table.mint([1], "cgt_support_split", [1]) - weights[ids == copy] *= 1.0 + 1e-12 + weights[ids == (1 if member == "root" else copy)] *= 1.0 + 1e-12 receipt, problems = _exhausting_receipt( _rebuild(frame, household_weights=weights), _NESTING_SCENARIO @@ -1708,6 +1712,259 @@ def without_the_check(values, *, target, whole, keeps_the_split=None): "support_split_selection_stored": {"missing": 0, "extra": 1} } + def test_a_household_the_correction_would_move_the_walk_past_keeps_it_green( + self, monkeypatch + ) -> None: + # The one-copy household sits an ulp below its band's support mass of + # 40, and the correction it could carry, which still counts one copy, + # lifts it onto 40, where the walk rerun stops at it + # (test_uk_cgt_support.py). The split's rerun of the walk passes it + # over, so the recompute divides the household the stage divided. + # With the rerun answering as the stage's walk did, it carries the + # correction, and the recompute misses that family. + from microcosm.build.uk_runtime import cgt_support + + scenario = { + "weights": [float(np.nextafter(40.0, -np.inf)), 82.4], + "wealth": [9.0, 8.0], + "incomes": [BAND_INCOMES[0]] * 2, + "top_cells": {0: {250_000: 16.0}}, + } + _, frame = _clone_first_split_then_cloned(1, scenario) + _, problems = _exhausting_receipt(frame, scenario) + assert problems == {} + + select = cgt_support.select_cgt_support_households + walks: list[np.ndarray] = [] + + def as_the_stage_walked(*args, **kwargs): + selected, rows = select(*args, **kwargs) + walks.append(selected) + return walks[0], rows + + monkeypatch.setattr( + cgt_support, "select_cgt_support_households", as_the_stage_walked + ) + _, unchecked = _clone_first_split_then_cloned(1, scenario) + monkeypatch.undo() + _, problems = _exhausting_receipt(unchecked, scenario) + assert len(walks) == 2 + assert problems == { + "support_split_selection_stored": {"missing": 1, "extra": 0} + } + + def test_at_one_clone_three_equal_halves_do_not_vouch_for_a_family( + self, + ) -> None: + # Every member of the 900 family (15 at 60) folds one float above 60, + # as the #970 anchor's roundings can leave them, and three of them + # show equal halves, as three members the corrections after the split + # moved could. Three corrections can account for three such members, + # so none of them vouches for w / n, and the hold reaches one float + # either side of the common weight: 900 divides by 15 to 60, within + # reach, and the stage's 15 stands. Were three equal-halves members + # enough to hold the family exactly, every weight dividing by 15 to + # the moved weight would count 16, and the family would be reported. + _, frame = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + members = np.r_[ + np.int64(1), copy_id_table.mint([1] * 14, "cgt_support_split", range(1, 15)) + ] + clones = copy_id_table.mint(members, "cgt_incidence_clone", 1) + moved = float(np.nextafter(60.0, np.inf)) + for position, (member, clone) in enumerate(zip(members, clones, strict=True)): + if position < _load_tool()._SUPPORT_SPLIT_CORRECTED_ROWS: + weights[ids == member] = moved / 2.0 + weights[ids == clone] = moved / 2.0 + else: + weights[ids == member] = 31.0 + weights[ids == clone] = moved - 31.0 + moved_frame = _rebuild(frame, household_weights=weights) + + _, problems, exact, within = _held_families(moved_frame, _NESTING_SCENARIO) + + assert problems == {} + assert (exact, within) == (5, 1) + + def test_the_hold_counts_every_weight_change_after_the_split(self) -> None: + """The K = 1 hold's premises, read from the spine graph and the + stages' code (microcosm#931 review). + + ``_SUPPORT_SPLIT_CORRECTED_ROWS`` counts the one-row exact-total + corrections that can land on a support family member between the split + and the tool's read, and ``_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS`` how + far the rest can move an uncorrected one. A spine node changes the + household weights only where it creates or expands rows or declares + weights: the executor refuses any other node's weights and frames. + After the split those are + the incidence clone, the #970 anchor and the residential split, and + the boundaries between them keep every row. The split, the clone and + the residential split each correct their total once, through + spi_support's helpers, which move one row; the anchor corrects none + and caps a clone half's factor at one, and refuses gains on a clone + member other than its carrier (test_uk_cgt_structure.py), so a liable + clone half takes one arm. Every later stage that changes no weight + builds its frames on its input's weights, so the legacy stage plan + moves none either. + """ + import ast + import inspect + + from microcosm.build.uk_runtime import ( + cgt_residential_split, + cgt_structure, + cgt_support, + graph_kernels, + spi_support, + ) + from microcosm.build.uk_runtime.cgt_structure import CGT_ANCHOR_MAXIMUM_FACTOR + from microcosm.build.uk_runtime.graph import uk_spine_graph + from microcosm.graph import StructuralDelta + from microcosm.graph.decl import compile_graph + + tool = _load_tool() + graph = uk_spine_graph() + order = list(compile_graph(graph).order) + later = order[order.index("cgt_support_split") + 1 :] + weighing = [ + node_id + for node_id in later + if graph.node(node_id).structural + in {StructuralDelta.CREATE, StructuralDelta.EXPAND} + or graph.node(node_id).weights is not None + ] + assert weighing == [ + "cgt_incidence_clone", + "cgt_incidence_anchor", + "cgt_residential_split", + ] + boundaries = { + graph.node(node_id).kernel + for node_id in later + if graph.node(node_id).structural is StructuralDelta.FILTER + } + assert boundaries == {"uk.identity@1"} + + # Each call that corrects a frame's total, by the function making it: + # a call to spi_support's helpers, or to a function of the stage's + # module that wraps them. + helpers = {"_exact_total_correction", "_importance_weights_with_exact_total"} + stages = ("cgt_support_split", *weighing) + modules = {graph_kernels._stage_module(stage) for stage in stages} + assert modules == {cgt_support, cgt_structure, cgt_residential_split} + corrections: dict[tuple[str, str], int] = {} + for module in modules: + tree = ast.parse(inspect.getsource(module)) + functions = { + node.name: node + for node in tree.body + if isinstance(node, ast.FunctionDef) + } + + def called(function) -> list[str]: + return [ + sub.func.id + for sub in ast.walk(function) + if isinstance(sub, ast.Call) and isinstance(sub.func, ast.Name) + ] + + wrappers = { + name + for name, function in functions.items() + if name.startswith("_exact") and set(called(function)) & helpers + } + for name, function in functions.items(): + if name in wrappers: + continue + count = sum(item in helpers | wrappers for item in called(function)) + if count: + corrections[(module.__name__, name)] = count + assert corrections == { + (cgt_support.__name__, "split_cgt_support_households"): 1, + (cgt_structure.__name__, "clone_cgt_incidence"): 1, + (cgt_residential_split.__name__, "split_cgt_residential_households"): 1, + } + assert len(corrections) == tool._SUPPORT_SPLIT_CORRECTED_ROWS + # spi_support's helpers move one row each. + rng = np.random.default_rng(3) + for _ in range(20): + values = rng.lognormal(3.0, 1.0, 50) + target = float(values.sum()) * (1.0 + 1e-15) + for corrected in ( + spi_support._exact_total_correction( + values, target=target, correction_index=int(np.argmax(values)) + ), + np.asarray( + spi_support._importance_weights_with_exact_total( + values, target + ).values + ), + ): + if corrected is not None: + assert int((corrected != values).sum()) <= 1 + # The anchor scales a clone half by at most one. + assert CGT_ANCHOR_MAXIMUM_FACTOR == 1.0 + assert tool._SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS == 1 + + # The stages after the split that change no weight pass their input's + # household weights to every frame they build. + passed = { + "frame.weights_for('household').values", + "np.asarray(frame.weights_for('household').values, dtype=float)", + } + + def binding(assignments, name: str, line: int): + """The value last bound to ``name`` before ``line``.""" + + found = [ + value + for lineno, target, value in assignments + if target == name and lineno < line + ] + return found[-1] if found else None + + built: dict[str, int] = {} + for stage in later: + if stage in weighing or stage not in graph_kernels._STAGE_MODULES: + continue + built[stage] = 0 + module = graph_kernels._stage_module(stage) + tree = ast.parse(inspect.getsource(module)) + for function in ast.walk(tree): + if not isinstance(function, ast.FunctionDef): + continue + assignments = sorted( + ( + (sub.lineno, sub.targets[0].id, sub.value) + for sub in ast.walk(function) + if isinstance(sub, ast.Assign) + and len(sub.targets) == 1 + and isinstance(sub.targets[0], ast.Name) + ), + key=lambda item: item[0], + ) + for sub in ast.walk(function): + if not isinstance(sub, ast.Call): + continue + name = getattr(sub.func, "id", getattr(sub.func, "attr", None)) + assert name not in helpers, (stage, function.name) + for keyword in sub.keywords: + if keyword.arg != "household_weights": + continue + value = keyword.value + if isinstance(value, ast.Name): + value = binding(assignments, value.id, sub.lineno) or value + elif isinstance(value, ast.Attribute) and isinstance( + value.value, ast.Name + ): + base = binding(assignments, value.value.id, sub.lineno) + if base is not None: + value = ast.Attribute(base, value.attr, ast.Load()) + assert ast.unparse(value) in passed, (stage, function.name) + built[stage] += 1 + assert len(built) == 9 and all(built.values()), built + # --- copy ids at the copy-id table's declared radices (microcosm#931) ---------- # diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 7ca738a9ee..5672680fad 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -1137,6 +1137,8 @@ def recompute(person_t, household_t, weights_t): #: incidence clone's and the residential split's. The #970 anchor applies #: none. So at most this many support-family members carry a corrected #: weight, and a family of more members has one that carries none. +#: test_uk_identity_stability_receipts derives both counts' premises from +#: the spine graph and the stages' code. _SUPPORT_SPLIT_CORRECTED_ROWS = 3 #: How many floats the #970 anchor and the residential arms' fold can move #: the pair sum of a K = 1 family member no correction touched. @@ -1987,9 +1989,11 @@ def plans_equal(left: pd.Series, right: pd.Series) -> bool: "support-split mass record: new_total against old_total at rtol " "1e-9; support-split copy counts: at K = 1, for a family of more " "than three members whose clone pair sums are all one weight c, " - "exactly the count of a weight w with fl(w / n) == c, or with " - "fl(w / n) within one float of c where fewer than four members " - "show their clone pair untouched; for every other family and at " + "exactly the count of a weight w with fl(w / n) == c, or, where " + "fewer than four members show their clone pair untouched, with " + "fl(w / n) within one float of c, a reach that can take in the " + "multiple of 60 itself and then does not report a family split " + "consistently into one copy more; for every other family and at " "K > 1, between the counts at the folded family weight times " "1 - 1e-9 and 1 + 1e-9, which for a family whose source-scaled " "weight K x w lies within a relative 1e-9 of a multiple of 60 (at " From c014d9a148586747922eaee0563dcb07e1b42267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:53:50 +0000 Subject: [PATCH 034/108] Name stage 42's one-copy check and the fallback's limit in its declarations (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María ruled on 9 Nov that stage 42's one-copy check (keeps_the_split) stays, as a reading of the whole-households-first rule, and that copy_rule names it. She also accepted that the shared fallback does not make the check, with the stage notes saying so. This commit changes declarations, text and tests only, not behaviour. - copy_rule gains one clause after the shared exception, built on the cap as that exception is (CGT_SUPPORT_ONE_COPY_CHECK). It reads the same in the code dictionary, sources.yaml and source_stages.json: "...; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The closing words keep the clause true where the fallback lands on a household the check passed over. Stage 26's copy_rule is unchanged, since it makes no such check. - Stage 42's notes gain one sentence: the shared fallback does not make the check, so where none of the 64 can carry the correction, it can put it on a one-copy household the check passed over. The identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No other stage's notes change, and the was_wealth and was_lisa notes are untouched. - The changelog fragment's Declarations bullet quotes the new copy_rule. Its Exact total bullet names the check in the same words and gives both synthetic cases. - Tests: the copy_rule pin and the packaged-declarations test end on the new clause. A new test pins the accepted case on two synthetic K = 1 frames where the one household left whole is selected at one copy and is the heaviest row. In the first it sits an ulp below its band's support mass of 40 (2 x 1.25 x 16), so its corrected weight would stop the walk at it. In the second it weighs 60, so its corrected weight counts two copies. In both the check passes it over, and the fallback gives it the weight the check refused, bit for bit as the rule before did. A fallback that made the check fails the test. What moves: - No value. On the two frames the stage's output equals the parent's and af667b7e8's bit for bit. The identity tool reports support_split_selection_stored {missing: 1} and {extra: 1}, as it did at af667b7e8. No H2 value moves. - Stage 42's stage contract digest, with its copy_rule and notes. - The implementation hashes of stages 42 and 26. cgt_support.py's bytes moved, and high_value_home_support binds it. Node keys chain, so every spine node from stage 26 on re-keys (43 of the 79 hermetic spine nodes; the 36 before it keep their keys), and so does every node downstream of the spine (75 of the 111 nodes of the full graph with its gates). - Every node keyed on the UK country spec's or solve's fingerprint, since sources.yaml, source_stages.json and the coverage manifest moved. Of the 99 kernel implementation hashes, 13 move: the two stages, plus the spine, full, preflight and national gates, the rotated holdout, the measures, the problem, the target compilation and selection, and the national targets and problem. With the spine gates attached, every spine node from the assembled gate on re-keys (64 of 81). - Regenerated: source_stages.json (stage 42's copy_rule and notes), the coverage manifest (25 source digests) and the H2 fixture (the same two lines in sources/fixture.json, and stage 42's stage_contract_sha256 in uk_spine.json). Tests: all synthetic and engine-free except H2. test_uk_cgt_support and 15 related files that read the declarations, the projection, the coverage manifest, the fingerprints or the stage's code pass (950 tests). Ruff check and format are clean on the two edited Python files. source_stages.json equals the typed projection, and the coverage manifest is current. Through the engine slot, the H2 fixture regenerates, its --check regenerates it byte-identically, and test_h2_uk_spine_parity passes. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 4 +- .../uk/release_input_coverage_manifest.json | 50 +++++----- .../src/microcosm/build/uk/source_stages.json | 4 +- .../src/microcosm/build/uk/spec/sources.yaml | 4 +- .../microcosm/build/uk_runtime/cgt_support.py | 22 ++++- .../engine_free/uk/test_uk_cgt_support.py | 97 ++++++++++++++++++- .../parity/uk_spine/sources/fixture.json | 4 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- 8 files changed, 146 insertions(+), 41 deletions(-) diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index bfead53b14..99c85ab393 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,6 +1,6 @@ The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. - **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where an exact-total correction fell on a member, and at K > 1 by more than the snap absorbs. So the count check does not recount the fold. At K = 1 a family of more than three members whose clone pair sums are all one weight c is held exactly: its stored count n must be the count of a weight w with fl(w / n) = c. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so such a family has a member no correction moved, whose pair sum only the later stages' roundings can have moved off w / n. Where at least four members show their two clone halves at equal weights, one of those is uncorrected, and equal halves mean the #970 anchor and the residential arms left its pair sum at exactly w / n, so c is exactly w / n. Elsewhere the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member. Every other family, and every family at K > 1, keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. That bracket opens for a family whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a family the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to at most the multiple itself, or the multiple and the float either side of it where the hold reaches one float. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. -- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception. Both stages' notes say where the correction lands. The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. -- **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy carries it only where its corrected weight leaves the walk's selection and its one copy as they were, so wherever one of the 64 carries it, the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the identity tool's recompute would have found a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy, then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. The fallback does not consult that check, so its row can also be a whole household, among them a one-copy household the check passed over; the rerun can then divide a different set. That can happen on synthetic frames whose one-copy household sits one to three ulps below its band's support mass, as it could under the rule before, and the identity tool then reports a missing family. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate unless a selected household left at one copy outweighs it, or ties it from an earlier row, which needs every unselected household to weigh at most 60 / K (to within the K-ulp snap at K > 1). The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. A selected household left at one copy that can carry it also costs a rerun of the selection walk, about 0.41 s at that size, so 64 that the check passes over would take about 26 s; a national frame does not reach that, since its 64 heaviest whole households are unselected ones, far heavier than 60 / K. +- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception and no more, since it makes no one-copy check. Both stages' notes say where the correction lands, and the CGT stage's add that the shared fallback does not make the one-copy check, and what the identity tool's selection recompute can then report. The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. +- **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy can carry it only where its corrected weight keeps its one copy and the walk's selection, the one-copy check `copy_rule` names, kept as a reading of the whole-households-first rule, so wherever one of the 64 carries it, the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the identity tool's recompute would have found a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy, then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. The fallback does not consult that check, so its row can also be a whole household, among them a one-copy household the check passed over; the rerun can then divide a different set. That can happen on synthetic frames, as it could under the rule before, with the weights the rule before gave: where the one-copy household sits one to three ulps below its band's support mass, the identity tool then reports a missing family, and where it weighs 60 at K = 1 and outweighs every copy, a family the stage never made. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate unless a selected household left at one copy outweighs it, or ties it from an earlier row, which needs every unselected household to weigh at most 60 / K (to within the K-ulp snap at K > 1). The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. A selected household left at one copy that can carry it also costs a rerun of the selection walk, about 0.41 s at that size, so 64 that the check passes over would take about 26 s; a national frame does not reach that, since its 64 heaviest whole households are unselected ones, far heavier than 60 / K. - **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot and a later one of the 64 can, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them; where none of the 64 can, nothing moves. At stage 42, and at K = 1 too, a selected household left at one copy that outweighs the heaviest unselected household, or ties it from an earlier row, comes first, which needs every unselected household to be as light; where it can carry the correction and keep the split, it takes it even where the heaviest unselected household could have, so those two rows' weights move by rounding amounts. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the K = 1 hold and the count bracket. Against the exact recount it replaces, at K = 1 it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, where its members fold to one weight and at least four of them show their clone pairs untouched, for all but 31 of the 9,995 counts n from 4 to 9,998. Where fewer do, the hold reaches one float, which takes in the multiple itself for 6,393 of those counts, so such a family goes unreported there, as the full-chain test's 900 family does. It no longer reports one of three or fewer members, or one whose members fold to different weights. At K > 1 no such family is reported. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`, and node keys chain on their inputs' keys, so every spine node from stage 26 on, and every node downstream of the spine, re-keys with them. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 91e3b27d1c..66b2f5ad38 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "8bce7eedaefd2d0e5f753e6c99c711af7a46ac472eb098059b0b67214bbaad8a", + "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index b60077decf..eae1a1ef56 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -7403,7 +7403,7 @@ ], "selection": "within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "maximum_copy_weight": 60.0, - "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make", "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "flag_column": "household_is_cgt_support_copy", "copies_column": "cgt_support_copies", @@ -7421,7 +7421,7 @@ "cgt_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." }, { "stage": "cgt_incidence_clone", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index ec2c492ff3..6a071b24ba 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -6015,7 +6015,7 @@ stages: - non_residential_property_value selection: within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused maximum_copy_weight: 60.0 - copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight + copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make id_remapping: copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused flag_column: household_is_cgt_support_copy copies_column: cgt_support_copies @@ -6030,7 +6030,7 @@ stages: - cgt_support_copies - cgt_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' + notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool''s selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' - stage: cgt_incidence_clone survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index 6b8f06796a..6079801f5a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -136,13 +136,30 @@ CGT_SUPPORT_CORRECTION_CANDIDATES = 64 #: The one exception to "every copy carries weight / copies" that the #: ``copy_rule`` of both support splits declares, built on the cap so the -#: declarations restate it (microcosm#931). +#: declarations restate it (microcosm#931). The CGT split's adds +#: :data:`CGT_SUPPORT_ONE_COPY_CHECK` after it. CGT_SUPPORT_EXACT_TOTAL_EXCEPTION = ( f"except where none of the {CGT_SUPPORT_CORRECTION_CANDIDATES} heaviest " "households left whole can carry the exact-total correction, when the " "shared fallback can put it on a root or copy, off weight / copies by " "about one ulp of the total weight" ) +#: The clause the CGT split's ``copy_rule`` adds after that exception, built +#: on the cap too. Of those candidates, a selected household left at one copy +#: can carry the correction only where its corrected weight keeps its one +#: copy and the walk's selection (``keeps_the_split`` in +#: :func:`split_cgt_support_households`, kept as a reading of the +#: whole-households-first rule). The shared fallback does not make that +#: check, so it can still put the correction on a one-copy household the +#: check passed over (María's rulings of 2026-11-09; microcosm#931). The +#: high-value home support split makes no such check, so its ``copy_rule`` +#: ends at the exception. +CGT_SUPPORT_ONE_COPY_CHECK = ( + f"of those {CGT_SUPPORT_CORRECTION_CANDIDATES}, a selected household left " + "at one copy can carry the correction only where its corrected weight " + "keeps its one copy and the walk's selection, a check the fallback does " + "not make" +) _HOUSEHOLD_SUPPORT_CHANNEL_COLUMN = "household_support_channel" #: The geographic clone index the clone-first spine gives every household #: (``rowwise_dataset.ladder_clone_index_column("household")``, the @@ -927,7 +944,8 @@ def cgt_support_split_operation_parameters() -> tuple[ "maximum_copy_weight), the quotient snapped to the nearest " "integer within clone-count ulps when the clone count " "exceeds one; every copy carries weight / copies, " - f"{CGT_SUPPORT_EXACT_TOTAL_EXCEPTION}" + f"{CGT_SUPPORT_EXACT_TOTAL_EXCEPTION}; " + f"{CGT_SUPPORT_ONE_COPY_CHECK}" ), "id_remapping": ( "copy-id table field cgt_support_split; copy k takes every " diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index a729036684..538bf86425 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -33,6 +33,7 @@ CGT_SUPPORT_MASS_CHANGE_REASON, CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, CGT_SUPPORT_MINIMUM_GAIN_BAND_LOWER, + CGT_SUPPORT_ONE_COPY_CHECK, CGT_SUPPORT_PUBLISHED_TOP_BAND_TAXPAYERS, CGT_SUPPORT_SPLIT_OPERATION_KIND, CGT_SUPPORT_SPLIT_STAGE_NAME, @@ -1121,7 +1122,8 @@ def test_split_records_pool_exhaustion_and_empty_bands_without_refusing(): # the unselected ones and the selected ones left at one copy. The correction # is tried on the 64 heaviest of them, heaviest first, and spi_support's # shared fallback, whose row can be a root or copy, runs only where none of -# them can carry it. +# them can carry it. The fallback does not make the one-copy check, so its +# row can also be a one-copy household the check passed over. @dataclass(frozen=True) @@ -1332,6 +1334,80 @@ def test_a_household_whose_correction_would_move_the_walk_is_passed_over( assert weights[1] == pytest.approx(41.2, rel=1e-12) +def test_the_fallback_does_not_make_the_one_copy_check(monkeypatch): + # copy_rule names the one-copy check and that the shared fallback does + # not make it (microcosm#931). In both frames the one household the split + # leaves whole is selected at one copy and is the heaviest row, so where + # the check passes it over, the fallback's first candidate is that same + # household, and it gets the very weight the check refused. The weights + # are the rule before's bit for bit, since no household is unselected, + # and the rule rerun on the output divides a different set. + # + # The walk: the first household sits an ulp below its band's support + # mass of 40 (2 x 1.25 x 16) and the second, divided into two copies of + # 32.075, crosses. The corrected weight still counts one copy but reaches + # 40, where the walk rerun stops at it, so the rerun on the output misses + # the second's family. + below = float(np.nextafter(40.0, -np.inf)) + frame = support_frame( + weights=[below, 64.15], wealth=[9.0, 8.0], incomes=[BAND_INCOMES[0]] * 2 + ) + result, call = _recorded_correction(monkeypatch, frame, {0: {250_000: 16.0}}) + + assert call.selected.tolist() == [True, True] + assert call.whole.tolist() == [0] + assert int(np.argmax(call.values)) == 0 + corrected = _exact_total_correction( + call.values, target=call.target, correction_index=0 + ) + assert corrected is not None and corrected[0] >= 40.0 + assert cgt_support_copy_counts([corrected[0]], clone_count=1).tolist() == [1] + (reselected,) = call.reselected + assert reselected.tolist() == [True, False] + assert _moved_rows(result, call) == [0] + weights = np.asarray(result.frame.weights_for("household").values) + assert weights[0] == corrected[0] + assert weights[1:].tolist() == [64.15 / 2] * 2 + before, carried = _correction_before(call) + assert not carried + assert weights.tolist() == before.tolist() + + # The count: all three are selected, and the 60 household, left at one + # copy, outweighs every copy of the other two. Its corrected weight lifts + # it above 60, where the rule counts two copies, so the check passes it + # over without a rerun of the walk, and the rerun on the output divides + # it, a family the stage never made. + frame = support_frame( + weights=[400.0, 268.7, 60.0], + wealth=[5.0, 6.0, 7.0], + incomes=[BAND_INCOMES[0]] * 3, + ) + result, call = _recorded_correction( + monkeypatch, + frame, + {0: {250_000: 1_000.0}}, + headroom=SCENARIO_HEADROOM, + ) + + assert call.selected.all() + assert call.whole.tolist() == [2] + assert int(np.argmax(call.values)) == 2 + corrected = _exact_total_correction( + call.values, target=call.target, correction_index=2 + ) + assert corrected is not None + assert cgt_support_copy_counts([corrected[2]], clone_count=1).tolist() == [2] + assert call.reselected == () + assert _moved_rows(result, call) == [2] + weights = np.asarray(result.frame.weights_for("household").values) + assert weights[2] == corrected[2] + household = result.frame.table("household") + assert household[CGT_SUPPORT_COPIES_COLUMN].tolist()[:3] == [7, 5, 1] + before, carried = _correction_before(call) + assert not carried + assert weights.tolist() == before.tolist() + + def test_a_one_copy_household_tied_with_the_heaviest_unselected_takes_it_first( monkeypatch, ): @@ -2064,7 +2140,10 @@ def test_operation_dictionary_restates_the_reviewed_design(): "carries weight / copies, except where none of the 64 heaviest " "households left whole can carry the exact-total correction, when the " "shared fallback can put it on a root or copy, off weight / copies by " - "about one ulp of the total weight" + "about one ulp of the total weight; of those 64, a selected household " + "left at one copy can carry the correction only where its corrected " + "weight keeps its one copy and the walk's selection, a check the " + "fallback does not make" ) assert payload["flag_column"] == HOUSEHOLD_IS_CGT_SUPPORT_COPY assert payload["copies_column"] == CGT_SUPPORT_COPIES_COLUMN @@ -2260,8 +2339,10 @@ def test_packaged_manifest_declares_the_stage() -> None: def test_the_packaged_declarations_restate_the_copy_rule() -> None: - # The copy rule names its one exception identically in the code - # dictionary, sources.yaml and source_stages.json (microcosm#931). + # The copy rule names its one exception, and then the one-copy check and + # that the fallback does not make it, identically in the code + # dictionary, sources.yaml and source_stages.json, both clauses built on + # the cap (microcosm#931). ((_, payload),) = cgt_support_split_operation_parameters() stage = load_country_spec("uk").sources.stage_map()[CGT_SUPPORT_SPLIT_STAGE_NAME] (operation,) = stage.operations @@ -2271,5 +2352,11 @@ def test_the_packaged_declarations_restate_the_copy_rule() -> None: (entry,) = [row for row in packaged["stages"] if row["stage"] == stage.stage] assert entry["operations"][0]["copy_rule"] == payload["copy_rule"] assert payload["copy_rule"].endswith( - "; every copy carries weight / copies, " + CGT_SUPPORT_EXACT_TOTAL_EXCEPTION + "; every copy carries weight / copies, " + + CGT_SUPPORT_EXACT_TOTAL_EXCEPTION + + "; " + + CGT_SUPPORT_ONE_COPY_CHECK + ) + assert CGT_SUPPORT_ONE_COPY_CHECK.startswith( + f"of those {CGT_SUPPORT_CORRECTION_CANDIDATES}, " ) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 1ba528d30c..faec8df43e 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -563,7 +563,7 @@ ], "grain": "household", "nonnegative_outputs": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", "operations": [ { "adult_minimum_age": 16, @@ -573,7 +573,7 @@ "conservation": "exact_total", "copies_column": "cgt_support_copies", "copy_index_column": "cgt_support_copy_index", - "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make", "declared_factor": 1, "expected_support_mass": 145000, "flag_column": "household_is_cgt_support_copy", diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index f03d9336c5..9d7e2f13f4 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"ed46fd4d2dc5df11ea9fdd0e955d8a7e13c20be5bae275b7a0a2f209a653fc78","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"71433baf763eb12157952507a3b605fd84b45045c75315191d12044d1b97cf4a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"ed46fd4d2dc5df11ea9fdd0e955d8a7e13c20be5bae275b7a0a2f209a653fc78","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"0edd2cdc89dbbb4fc9ad5332db4484522904f3bcede684cb89b189e99eafc537","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} From 6e42fa103e952cdee71e1cfc081c4bc392e6b5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:04:17 +0000 Subject: [PATCH 035/108] Hold the support-copy count on the members with untouched clone halves, at any clone count (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María's 9 Nov evening rulings on the identity tool's count hold: keep its two narrowings, hold a family on the members whose clone halves are still equal, extend the hold to K > 1, and hold the family test to the same tolerance as the count. Tool only; no stage code, value, declaration or node key moves. - Equal halves (ruling 3). Where at least four members of a family of more than three show their two clone halves at equal weights and share one pair sum c, the stored count n is held exactly on c, whatever the other members carry (trimmed or liable members, a corrected one). At most three members carry a corrected weight, so one of the four is uncorrected, and equal halves leave its pair sum at exactly w / n. Before, a family was held only where every member carried c. Both accepted narrowings stay: families of three or fewer keep the bracket, and where every member shares c but fewer than four show equal halves, the hold reaches one float either side of c. - K > 1 (ruling 4). The stage counts n on each clone's own weight and gives every member fl(w / n), and the snapped count still rises with w, so the hold applies at any K, with the K-aware count at the ends of the run of weights. The one-float tier reaches two floats where c sits within two floats of a power of two. c lies in (45 / K, 60 / K], which holds 16 at K = 3, 8 at K = 6 and 7, and 4 at K = 12 to 15, and below a power of two the floats are half as wide. The hold's docstring gives the bound: a trimmed pair's sum lies within three quarters of the float spacing at c before its last rounding, a liable pair's within half. - The family test (ruling 3). Whether a household is a family is no longer the exact recount of the re-summed fold. A household the rule selects is possibly a family where its K-aware count at the folded weight times 1 + 1e-9 exceeds one, and possibly whole where its count times 1 - 1e-9 is one. Its stored state is accepted in between, and the hold's counts decide where the hold applies. The keys and their missing and extra counts are unchanged. The hold is now asked about every stored family, so it takes only a positive normal pair sum: a family of zero-weight members falls to the bracket and is reported missing, where the run of weights would step below zero and the tool would raise. - The premises test (the re-review's first nit) counts the exact-total corrections in every function, method and nested function of the three weighing stages, and matches attribute calls as well as bare names. - Texts: the _e8_support_split docstring, the hold's docstrings and constants, the count-check comment, the receipt's tolerance_policy, the receipts test's class docstring and the changelog fragment (Receipt and gate, What moves, and one Exact total sentence this commit makes stale). What moves: tool verdicts and the tolerance_policy text only. The receipt's keys are unchanged, and copies_recomputed is still the exact recount. No node or fingerprint hashes tools/, and nothing is regenerated. - At K > 1 the count check now reports a family the stage split consistently into one copy more on a multiple. With the floor(q) + 1 stage on the nesting scenario it reports all four families in every clone at K = 2, 3, 4, 6, 7, 11, 13 and 15. The parent tool reported none. - At K = 1 it also reports such a family where four members show equal halves and others differ. A family with one corrected member now reports. - The family test no longer reports a family missing, or extra, where a household's fold crosses 60 / K from its stored state. That removes the 9 Nov probe's false missing family (a two-copy family a float above 60 whose fold lands on 60) and one false extra at K = 2. It also no longer reports a household split into two copies at 60 / K (the floor(q) + 1 stage's two-copy households), nor stage 42's one-copy household that the shared fallback lifts across 60. The rule rerun still counts the latter in copies_recomputed. Ruling 2's walk-moving case is still reported, on all seven of its frames. Synthetic evidence, counts only, against a git-archive snapshot of 8ea25b96b's tool: - No false support_split_copies_stored or support_split_selection_stored on frames the real stages build, at K = 1, 2, 3, 4, 6, 7, 11, 13 and 15: - the 8 Nov tool_probe classes and the nesting scenario, split then cloned, where tampered counts get the parent's verdicts; - 30 random frames with source weights on and one to three ulps above multiples of 60, and 20 with clone weights on and above multiples of 60 / K; - the nesting scenario and 12 or 24 random frames through a hand redraw, the anchor and the residential split, in two to four gain mixes; - frames whose c sits at or within three floats of 16, 8 or 4 at K = 3, 6, 7, 12, 13 and 15, as built, with every member moved one float off c (two below where c is the power) and through the full chain. - Families held: 83 to 84 percent exactly, split then cloned. Through the full chain, 64 to 74 percent exactly and 7 to 13 percent within the reach; at K = 15, 927 of 33,600 within two floats. - Detection: floor(q) + 1 gives 2,350 to 20,628 count flags split then cloned at K > 1, and 7,934 to 21,217 through the full chain. The parent gives none. At K = 1 the full chain gives 3,258 against the parent's 1,322. Every floor(q) + 1 frame is flagged. - The false missing family's frame is green, where the parent reports {missing: 1}. - The pair arithmetic replayed at 11 clone counts from 1 to 15, 2.4 to 4.8 million members per K and case, 0.7 to 1.3 million of them within two floats of a power of two where (45 / K, 60 / K] holds one: never more than one float off c, and equal halves never moved. Across 267,035 families near a power of two, a two-float reach never accepts a count one float rejects. - The blind spot, by design arithmetic over every n from 4 to 9,998. At K = 1 it is as before. At K = 2 to 15 the snapped count's step sits 3 to 17 floats above the multiple of 60 / K. Held exactly, at most the float either side of the step, for up to 2,975 of the 9,995 counts. In the one-float tier, up to three floats from it. floor(q) + 1 on a clone of a source multiple is reported for every count where it changes the count. - Mutations: the premises test now catches a second correction written as spi_support._importance_weights_with_exact_total(...) in clone_cgt_incidence, a bare-name one in UKCGTIncidenceCloneStageTransform.__call__, and a wrapper call in a nested function. The test before let the first two through. Eight of nine tool mutations fail the receipts tests. The ninth, not letting the held counts decide the family test, changes no verdict: a held family's fold is n times its shared pair sum. Tests (engine-free, synthetic): floor(q) + 1 is reported at K = 3, 7, 13 and 15; a family with a corrected member is held on its equal halves; four equal halves vouch for a family whose others differ, three do not; families of three or fewer stay on the bracket at K = 1 and 3; through the anchor and the residential split the correct stage stays green at K = 1 and 3 and floor(q) + 1 reports 3 and 12; members near a power of two are held to two floats; the two-copy family that folds onto 60 is green; a family dropped or added away from a multiple is reported at K = 1 and 3; a zero-weight family is reported, not refused; the stage-42 one-copy check case is now green, with its stage weight checked instead. The seven engine-free files that load the tool pass (257 tests), and ruff is clean on both Python files. source_stages.json, the coverage manifest and ci_test_plan verify are current. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 6 +- .../uk/test_uk_identity_stability_receipts.py | 550 ++++++++++++++---- tools/verify_uk_identity_stability.py | 388 +++++++----- 3 files changed, 681 insertions(+), 263 deletions(-) diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index 99c85ab393..6ac7294a1e 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,6 +1,6 @@ The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. -- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: at K = 1 for a weight a few ulps above a multiple of 60, or where an exact-total correction fell on a member, and at K > 1 by more than the snap absorbs. So the count check does not recount the fold. At K = 1 a family of more than three members whose clone pair sums are all one weight c is held exactly: its stored count n must be the count of a weight w with fl(w / n) = c. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so such a family has a member no correction moved, whose pair sum only the later stages' roundings can have moved off w / n. Where at least four members show their two clone halves at equal weights, one of those is uncorrected, and equal halves mean the #970 anchor and the residential arms left its pair sum at exactly w / n, so c is exactly w / n. Elsewhere the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member. Every other family, and every family at K > 1, keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. That bracket opens for a family whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a family the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to at most the multiple itself, or the multiple and the float either side of it where the hold reaches one float. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. +- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: for a weight a few ulps above a multiple of 60 (at K > 1, by more than the snap absorbs), or where an exact-total correction or the #970 anchor's and the residential arms' roundings moved a member. So neither the count check nor the family test recounts the fold. At any K, a family of more than three members is held exactly where at least four of its members show their two clone halves at equal weights and share one pair sum c, whatever the others carry: its stored count n must be the count of a weight w with fl(w / n) = c. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so one of those four or more is uncorrected, and equal halves mean the anchor and the arms left its pair sum at exactly w / n, so c is exactly w / n. The stage counts n on each clone's own weight and gives every member fl(w / n), and the snapped count still rises with w, so this holds at K > 1 as at K = 1. Where every member shares c but fewer than four show equal halves, the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member where the floats either side of c are as wide, and two floats where c sits within two floats of a power of two, below which they are half as wide; c lies in (45 / K, 60 / K], which holds 16 at K = 3, 8 at K = 6 and 7, and 4 at K = 12 to 15. Every other family keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. The family test, which decides whether a household is a family at all, is held the same way at the K-aware count: a household the rule selects is possibly a family where its count at the folded weight times 1 + 1e-9 exceeds one and possibly whole where its count at the folded weight times 1 − 1e-9 is one, its stored state accepted in between, and the hold's counts decide where it applies. That bracket opens for a household whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies, a whole household or a family of two among them. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a household the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to a few floats next to the count's step, for some counts n. At K = 1 the step sits a float above the multiple, which leaves at most the multiple itself where the family is held exactly, or the multiple and the float either side of it where the hold reaches one float. At K from 2 to 15 the snap puts the step 3 to 17 floats above the multiple of 60 / K, which leaves at most the float on either side of the step where the family is held exactly, or up to three floats from it in the one-float tier, so a family on the multiple itself split into one copy more is reported. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. - **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception and no more, since it makes no one-copy check. Both stages' notes say where the correction lands, and the CGT stage's add that the shared fallback does not make the one-copy check, and what the identity tool's selection recompute can then report. The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. -- **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy can carry it only where its corrected weight keeps its one copy and the walk's selection, the one-copy check `copy_rule` names, kept as a reading of the whole-households-first rule, so wherever one of the 64 carries it, the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the identity tool's recompute would have found a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy, then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. The fallback does not consult that check, so its row can also be a whole household, among them a one-copy household the check passed over; the rerun can then divide a different set. That can happen on synthetic frames, as it could under the rule before, with the weights the rule before gave: where the one-copy household sits one to three ulps below its band's support mass, the identity tool then reports a missing family, and where it weighs 60 at K = 1 and outweighs every copy, a family the stage never made. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate unless a selected household left at one copy outweighs it, or ties it from an earlier row, which needs every unselected household to weigh at most 60 / K (to within the K-ulp snap at K > 1). The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. A selected household left at one copy that can carry it also costs a rerun of the selection walk, about 0.41 s at that size, so 64 that the check passes over would take about 26 s; a national frame does not reach that, since its 64 heaviest whole households are unselected ones, far heavier than 60 / K. -- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot and a later one of the 64 can, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them; where none of the 64 can, nothing moves. At stage 42, and at K = 1 too, a selected household left at one copy that outweighs the heaviest unselected household, or ties it from an earlier row, comes first, which needs every unselected household to be as light; where it can carry the correction and keep the split, it takes it even where the heaviest unselected household could have, so those two rows' weights move by rounding amounts. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the K = 1 hold and the count bracket. Against the exact recount it replaces, at K = 1 it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, where its members fold to one weight and at least four of them show their clone pairs untouched, for all but 31 of the 9,995 counts n from 4 to 9,998. Where fewer do, the hold reaches one float, which takes in the multiple itself for 6,393 of those counts, so such a family goes unreported there, as the full-chain test's 900 family does. It no longer reports one of three or fewer members, or one whose members fold to different weights. At K > 1 no such family is reported. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`, and node keys chain on their inputs' keys, so every spine node from stage 26 on, and every node downstream of the spine, re-keys with them. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. +- **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy can carry it only where its corrected weight keeps its one copy and the walk's selection, the one-copy check `copy_rule` names, kept as a reading of the whole-households-first rule, so wherever one of the 64 carries it, the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the rule rerun on the stage's output would have counted a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy, then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. The fallback does not consult that check, so its row can also be a whole household, among them a one-copy household the check passed over; the rerun can then divide a different set. That can happen on synthetic frames, as it could under the rule before, with the weights the rule before gave: where the one-copy household sits one to three ulps below its band's support mass, the identity tool then reports a missing family, and where it weighs 60 at K = 1 and outweighs every copy, the rule rerun counts a family the stage never made, which the tool's family test, held to its bracket, does not report, since the corrected weight lies within a relative 1e-9 of 60. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate unless a selected household left at one copy outweighs it, or ties it from an earlier row, which needs every unselected household to weigh at most 60 / K (to within the K-ulp snap at K > 1). The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. A selected household left at one copy that can carry it also costs a rerun of the selection walk, about 0.41 s at that size, so 64 that the check passes over would take about 26 s; a national frame does not reach that, since its 64 heaviest whole households are unselected ones, far heavier than 60 / K. +- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot and a later one of the 64 can, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them; where none of the 64 can, nothing moves. At stage 42, and at K = 1 too, a selected household left at one copy that outweighs the heaviest unselected household, or ties it from an earlier row, comes first, which needs every unselected household to be as light; where it can carry the correction and keep the split, it takes it even where the heaviest unselected household could have, so those two rows' weights move by rounding amounts. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the hold, the count bracket and the bracketed family test. Against the exact recount it replaces, it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count, nor a family missing, or extra, where a household's re-summed fold lands across the one-copy boundary (60 / K) from its stored state: on the synthetic frames, the one false missing family the full-chain probes found, a two-copy family a float above 60 that folds onto 60, and one false extra family at K = 2. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, where at least four of its members show equal halves sharing one weight, whatever the others carry: at K = 1 for all but 31 of the 9,995 counts n from 4 to 9,998, and at K from 2 to 15 for every count. Where fewer show equal halves but every member shares one weight, the hold reaches one float, which at K = 1 takes in the multiple itself for 6,393 of those counts, so such a family goes unreported there, as the full-chain test's 900 family does at one clone; at K > 1 it is reported. It no longer reports a family of three or fewer members, or one whose members fold to different weights without four equal halves among them, nor a household split into two copies at 60 / K, which the family test's bracket cannot tell from a whole one. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`, and node keys chain on their inputs' keys, so every spine node from stage 26 on, and every node downstream of the spine, re-keys with them. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 432b0dcab4..329ccba3d7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1199,14 +1199,15 @@ def _split_with_floor_plus_one(monkeypatch, clone_count: int, scenario): return _clone_first_split_then_cloned(clone_count, scenario) -def _held_families(frame, scenario) -> tuple[dict, dict, int, int]: - """The support-split receipt and problems, with the K = 1 hold's reach. +def _hold_reaches(frame, scenario, tool=None) -> tuple[dict, dict, dict[int, int]]: + """The support-split receipt and problems, with the hold's reach by family. - Also returns how many families the hold took exactly and how many to - within one float; at ``K > 1``, or where no family is held, both are 0. + The reach counts map how many floats the hold reached around a family's + shared pair sum (0 where it took the family exactly) to how many + families it held so; a family it did not hold is not counted. """ - tool = _load_tool() + tool = _load_tool() if tool is None else tool dividing_to = tool._floats_dividing_to reaches: list[np.ndarray] = [] @@ -1216,15 +1217,31 @@ def recorded(quotients, divisors, floats): tool._floats_dividing_to = recorded problems: dict[str, object] = {} - receipt = tool._e8_support_split( - frame, - problems, - distribution=support_distribution(scenario["top_cells"]), - parameters=PARAMETERS, - permutation_seed=7, - ) + try: + receipt = tool._e8_support_split( + frame, + problems, + distribution=support_distribution(scenario["top_cells"]), + parameters=PARAMETERS, + permutation_seed=7, + ) + finally: + tool._floats_dividing_to = dividing_to reach = np.concatenate(reaches) if reaches else np.zeros(0, dtype=np.int64) - return receipt, problems, int((reach == 0).sum()), int((reach > 0).sum()) + values, counts = np.unique(reach, return_counts=True) + return receipt, problems, dict(zip(values.tolist(), counts.tolist(), strict=True)) + + +def _held_families(frame, scenario) -> tuple[dict, dict, int, int]: + """The support-split receipt and problems, with the hold's reach. + + Also returns how many families the hold took exactly and how many to + within one float or two; where no family is held, both are 0. + """ + + receipt, problems, reaches = _hold_reaches(frame, scenario) + within = sum(count for reach, count in reaches.items() if reach > 0) + return receipt, problems, reaches.get(0, 0), within def _through_the_residential_split(twinned): @@ -1314,20 +1331,25 @@ class TestSupportSplitRecomputeAcrossGeographicClones: The stage reads ``K`` from its input and counts ``ceil(K x w / 60)`` copies (microcosm#931); the tool reads ``K`` from the pre-split table the same way. At one clone its receipt is the one the rule before ``K`` gave. - The fold re-sums the copies, so the count check never recounts it. At one - clone a family of more than three members whose clone pair sums are all - one weight ``c`` is held exactly: the stored ``n`` must be the count of a - weight that divides by ``n`` to ``c``, or to within one float of it where - fewer than four members show their clone pair untouched, a reach that can + The fold re-sums the copies, so neither the count check nor the family + test recounts it. At any clone count a family of more than three members + is held exactly where at least four of its members show their two clone + halves at equal weights and share one pair sum ``c``, whatever the others + carry: the stored ``n`` must be the count of a weight that divides by + ``n`` to ``c``. Where every member shares ``c`` but fewer show equal + halves, the weight may divide to within one float of ``c``, or two where + ``c`` sits that close to a power of two, a reach that at one clone can take in the multiple of 60 itself, where a family split consistently into - one copy more goes unreported. Every other family, and every family at - ``K > 1``, is held to the family weights' relative tolerance of 1e-9. - That bracket opens for a family whose - source-scaled weight ``K x w`` lies within a relative 1e-9 of a multiple - of 60 (at ``K = 1``, its own weight), where it cannot tell ``n`` copies - from ``n + 1``: a stored count moved on its own is left to the flags and - the family weights, and a family split consistently into one copy more - goes unreported. + one copy more goes unreported. Every other family, and whether each + household is a family at all, is held to the family weights' relative + tolerance of 1e-9, the counts at the folded weight times ``1 - 1e-9`` and + ``1 + 1e-9``, unless the hold decides. That bracket opens for a household + whose source-scaled weight ``K x w`` lies within a relative 1e-9 of a + multiple of 60 (at ``K = 1``, its own weight), where it cannot tell ``n`` + copies from ``n + 1``, or a family of two from a whole household: a + stored count moved on its own is left to the flags and the family + weights, and a household split consistently into one copy more goes + unreported. """ def test_at_one_clone_the_receipt_is_the_rule_before_the_clone_count( @@ -1410,19 +1432,24 @@ def test_exact_multiples_of_60_receipt_green_at_any_clone_count( # count's snap absorbs, so an exact recount would count 128 in all 11 # clones at 11 and in 12 of the 13 at 13 (the clone whose last copy # took the exact-total correction, which no household left whole - # could carry there, re-sums to 127); the tolerance accepts the - # stage's 127. + # could carry there, re-sums to 127). The hold takes every family + # exactly on its members' shared weight and accepts the stage's 127, + # except, at 7 and 13 clones, that last clone's 7,620 family: its + # corrected copy shows equal halves off the others' weight, so the + # family falls to the bracket, which accepts 127 too. one_split, one = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) split, many = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) one_receipt, one_problems = _exhausting_receipt(one, _NESTING_SCENARIO) - receipt, problems = _exhausting_receipt(many, _NESTING_SCENARIO) + receipt, problems, exact, within = _held_families(many, _NESTING_SCENARIO) assert one_problems == problems == {} assert one_receipt["families"] == 6 assert one_receipt["copies"] == one_split.copies_created == 259 assert receipt["families"] == clone_count * 6 assert receipt["copies"] == split.copies_created == clone_count * 259 + assert exact == clone_count * 6 - (clone_count in (7, 13)) + assert within == 0 @pytest.mark.parametrize("clone_count", [1, 3, 11]) def test_a_count_moved_off_a_non_multiple_is_reported( @@ -1448,10 +1475,10 @@ def test_a_count_moved_on_an_exact_multiple_is_left_to_the_flags( self, clone_count: int ) -> None: # Household 1 (900: 15 copies) moved to 16 on its root alone. Its 15 - # members no longer match that count, so even at one clone the family - # is not held exactly, and K x w / 60 is exactly 15, where the bracket - # cannot tell 15 from 16 (microcosm#931 review). The count check - # passes; the flags and the family weights report the family instead. + # members no longer match that count, so the family is not held at any + # clone count, and K x w / 60 is exactly 15, where the bracket cannot + # tell 15 from 16 (microcosm#931 review). The count check passes; the + # flags and the family weights report the family instead. _, frame = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) household = frame.table("household").copy() household.loc[household["household_id"] == 1, CGT_SUPPORT_COPIES_COLUMN] = 16 @@ -1485,20 +1512,27 @@ def test_at_one_clone_a_family_split_into_one_more_copy_on_a_multiple_is_reporte assert exact == receipt["families"] == 6 assert within == 0 - def test_beyond_one_clone_a_family_split_into_one_more_copy_on_a_multiple_goes_unreported( - self, monkeypatch + @pytest.mark.parametrize("clone_count", [3, 7, 13, 15]) + def test_beyond_one_clone_a_family_split_into_one_more_copy_on_a_multiple_is_reported( + self, monkeypatch, clone_count: int ) -> None: - # The blind spot the bracket leaves (microcosm#931 review): at three - # clones the same four families, split into one copy more in every - # clone, pass every check, since no family is held exactly at K > 1; - # only the exact recount records the gap. - split, frame = _split_with_floor_plus_one(monkeypatch, 3, _NESTING_SCENARIO) + # The same four families, split into one copy more in every clone. The + # stage counts n on each clone's own weight w and gives every member + # fl(w / n), so the hold applies at any clone count: each family's + # members still share one weight, every weight that divides by n + 1 + # to it counts n copies under the snapped count, and the hold reports + # all four in every clone. The bracket, which the tool before applied + # at K > 1, reported none. + split, frame = _split_with_floor_plus_one( + monkeypatch, clone_count, _NESTING_SCENARIO + ) - receipt, problems = _exhausting_receipt(frame, _NESTING_SCENARIO) + receipt, problems, exact, within = _held_families(frame, _NESTING_SCENARIO) - assert problems == {} - assert receipt["copies"] == split.copies_created == 3 * (259 + 4) - assert receipt["copies_recomputed"] < receipt["copies"] + assert problems == {"support_split_copies_stored": 4 * clone_count} + assert receipt["copies"] == split.copies_created == clone_count * (259 + 4) + assert exact == receipt["families"] == 6 * clone_count + assert within == 0 def test_at_one_clone_families_one_to_three_ulps_above_a_multiple_stay_green( self, @@ -1532,14 +1566,17 @@ def test_at_one_clone_families_one_to_three_ulps_above_a_multiple_stay_green( assert receipt["copies"] == split.copies_created == 244 assert receipt["copies_recomputed"] == 242 - def test_at_one_clone_a_family_with_a_corrected_member_falls_to_the_bracket( + def test_at_one_clone_a_family_with_a_corrected_member_is_held_on_its_equal_halves( self, monkeypatch ) -> None: # One copy of the 900 family, which the floor(q) + 1 stage split into - # 16, carries a rounding amount more, as where an exact-total - # correction falls on a member. Its members no longer carry one - # weight, so the family is held to the bracket, which cannot tell 15 - # from 16 at the multiple; the other three stay held and reported. + # 16, carries a rounding amount more on its original half, as where + # the incidence clone's exact-total correction falls on a member. Its + # members no longer carry one weight, but the other 15 still show + # their two clone halves at equal weights and share w / 16, so the + # hold takes the family exactly on them and reports it with the + # other three. The tool before held only families whose members all + # carried one weight, and left this one to the bracket. split, frame = _split_with_floor_plus_one(monkeypatch, 1, _NESTING_SCENARIO) ids = frame.table("household")["household_id"].to_numpy() weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() @@ -1550,19 +1587,75 @@ def test_at_one_clone_a_family_with_a_corrected_member_falls_to_the_bracket( _rebuild(frame, household_weights=weights), _NESTING_SCENARIO ) - assert problems == {"support_split_copies_stored": 3} - assert exact == 5 + assert problems == {"support_split_copies_stored": 4} + assert exact == 6 assert within == 0 - def test_at_one_clone_a_family_of_three_or_fewer_is_left_to_the_bracket( - self, monkeypatch + @pytest.mark.parametrize( + ("stage", "equal", "reported", "exact"), + [ + ("floor_plus_one", 4, 4, 6), + ("floor_plus_one", 3, 3, 5), + ("correct", 4, 0, 6), + ("correct", 3, 0, 5), + ], + ) + def test_at_one_clone_four_equal_halves_vouch_for_a_family_whose_others_differ( + self, monkeypatch, stage: str, equal: int, reported: int, exact: int + ) -> None: + # Every member of the 900 family but ``equal`` of them has its halves + # re-split unevenly with its pair sum one float above its stage-time + # weight, as the #970 anchor's roundings can leave a trimmed pair, so + # its members carry two weights. Four members left with equal halves + # sharing one weight vouch for w / n: at most three members carry a + # corrected weight, so one of the four does not, and the hold takes + # the family exactly on them. It then reports the floor(q) + 1 + # stage's 16 and accepts the correct stage's 15. With three left, any + # of them may be corrected, the family falls to the bracket, and the + # floor(q) + 1 family goes unreported with the bracket's blind spot. + if stage == "floor_plus_one": + _, frame = _split_with_floor_plus_one(monkeypatch, 1, _NESTING_SCENARIO) + count = 16 + else: + _, frame = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) + count = 15 + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + members = np.r_[ + np.int64(1), + copy_id_table.mint([1] * (count - 1), "cgt_support_split", range(1, count)), + ] + clones = copy_id_table.mint(members, "cgt_incidence_clone", 1) + for member, clone in list(zip(members, clones, strict=True))[equal:]: + moved = float( + np.nextafter( + weights[ids == member][0] + weights[ids == clone][0], np.inf + ) + ) + weights[ids == clone] = 16.0 + weights[ids == member] = moved - 16.0 + + _, problems, held_exactly, within = _held_families( + _rebuild(frame, household_weights=weights), _NESTING_SCENARIO + ) + + assert problems == ( + {"support_split_copies_stored": reported} if reported else {} + ) + assert held_exactly == exact + assert within == 0 + + @pytest.mark.parametrize("clone_count", [1, 3]) + def test_a_family_of_three_or_fewer_is_left_to_the_bracket( + self, monkeypatch, clone_count: int ) -> None: # Each of the three exact-total corrections after the split moves one # row, so in a family of three or fewer every member may carry a # corrected weight, and none vouches for w / n: the family is held to - # the bracket. Under the floor(q) + 1 stage the 120 family becomes - # three members and goes unreported; the 240 family becomes five and - # is reported. The correct stage receipts green. + # the bracket at any clone count. Under the floor(q) + 1 stage the + # 120 family becomes three members in every clone and goes + # unreported; the 240 family becomes five and is reported. The + # correct stage receipts green. assert _load_tool()._SUPPORT_SPLIT_CORRECTED_ROWS == 3 scenario = { "weights": [120.0, 240.0, 9_000.0], @@ -1570,16 +1663,16 @@ def test_at_one_clone_a_family_of_three_or_fewer_is_left_to_the_bracket( "incomes": [BAND_INCOMES[0]] * 2 + [BAND_INCOMES[37_700]], "top_cells": {0: {250_000: 1_000_000.0}}, } - _, correct = _clone_first_split_then_cloned(1, scenario) - _, mutated = _split_with_floor_plus_one(monkeypatch, 1, scenario) + _, correct = _clone_first_split_then_cloned(clone_count, scenario) + _, mutated = _split_with_floor_plus_one(monkeypatch, clone_count, scenario) _, correct_problems = _exhausting_receipt(correct, scenario) receipt, problems, exact, within = _held_families(mutated, scenario) assert correct_problems == {} - assert problems == {"support_split_copies_stored": 1} - assert receipt["families"] == 2 - assert exact == 1 + assert problems == {"support_split_copies_stored": clone_count} + assert receipt["families"] == 2 * clone_count + assert exact == clone_count assert within == 0 def test_at_one_clone_members_moved_one_float_are_held_to_within_one_float( @@ -1623,22 +1716,31 @@ def test_at_one_clone_members_moved_one_float_are_held_to_within_one_float( assert within == 1 assert exact_problems == {"support_split_copies_stored": 1} + @pytest.mark.parametrize( + ("clone_count", "reported"), + [(1, 3), (3, 12)], + ) def test_through_the_anchor_and_the_residential_split_the_hold_stays_sound( - self, monkeypatch + self, monkeypatch, clone_count: int, reported: int ) -> None: # The nesting scenario through the support split, the incidence # clone, a hand redraw giving the clone carriers gains in all four of # the anchor's groups, the #970 anchor and the residential split, with # the arms folded away as the E8 entry folds them. The anchor trims - # some pairs and the arms divide the liable clones, so two families - # show fewer than four untouched pairs (every carrier of the 900 - # family falls in one trimmed group) and are held to within one - # float; the other four are held exactly. The correct stage receipts - # green. Under the floor(q) + 1 stage three of the four families on a - # multiple are reported; the fourth, the 900 family, is held to - # within one float, a reach that takes in the multiple. - _, correct = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) - _, mutated = _split_with_floor_plus_one(monkeypatch, 1, _NESTING_SCENARIO) + # some pairs and the arms divide the liable clones, so in every clone + # two families show fewer than four equal halves and differ (every + # carrier of the 900 family falls in one trimmed group) and are held + # to within one float; the other four are held exactly on their equal + # halves. The correct stage receipts green. Under the floor(q) + 1 + # stage, at one clone three of the four families on a multiple are + # reported; the fourth, the 900 family, is held to within one float, + # a reach that takes in the multiple. At three clones all four are + # reported in every clone: the multiple sits a few floats below the + # snapped count's step, beyond that reach. + _, correct = _clone_first_split_then_cloned(clone_count, _NESTING_SCENARIO) + _, mutated = _split_with_floor_plus_one( + monkeypatch, clone_count, _NESTING_SCENARIO + ) correct_receipt, correct_problems, correct_exact, correct_within = ( _held_families(_through_the_residential_split(correct), _NESTING_SCENARIO) @@ -1648,46 +1750,126 @@ def test_through_the_anchor_and_the_residential_split_the_hold_stays_sound( ) assert correct_problems == {} - assert correct_receipt["families"] == 6 - assert (correct_exact, correct_within) == (4, 2) - assert problems == {"support_split_copies_stored": 3} - assert (exact, within) == (4, 2) + assert correct_receipt["families"] == 6 * clone_count + assert (correct_exact, correct_within) == (4 * clone_count, 2 * clone_count) + assert problems == {"support_split_copies_stored": reported} + assert (exact, within) == (4 * clone_count, 2 * clone_count) @pytest.mark.parametrize("member", ["root", "copy"]) def test_at_one_clone_a_family_folded_just_above_a_multiple_is_green( self, member: str ) -> None: # The root or one copy of household 1 (900: 15 copies of 60) carries a - # rounding amount more, as where an exact-total correction falls on a - # member, so the family folds to just above 900, where an exact - # recount gives 16. Its members no longer carry one weight, so the - # family is held to the bracket at one clone too (microcosm#931 - # review), which accepts the stage's 15, and the receipt is green; its - # exact recount still records the extra copy. Held exactly on the - # root's weight, the moved root would report the family. + # rounding amount more on its original half, as where the incidence + # clone's exact-total correction falls on a member, so the family + # folds to just above 900, where an exact recount gives 16. The moved + # member's halves differ, and the other 14 still show equal halves + # at 60, so the hold takes the family exactly on them and accepts the + # stage's 15, and the receipt is green; its exact recount still + # records the extra copy. Held exactly on the root's weight, the + # moved root would report the family. split, frame = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) ids = frame.table("household")["household_id"].to_numpy() weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() (copy,) = copy_id_table.mint([1], "cgt_support_split", [1]) weights[ids == (1 if member == "root" else copy)] *= 1.0 + 1e-12 - receipt, problems = _exhausting_receipt( + receipt, problems, exact, within = _held_families( _rebuild(frame, household_weights=weights), _NESTING_SCENARIO ) assert problems == {} assert receipt["copies"] == split.copies_created == 259 assert receipt["copies_recomputed"] == 260 + assert (exact, within) == (6, 0) + + @pytest.mark.parametrize( + ("clone_count", "source_weight", "copies", "moved_floats"), + [ + # 900 at 15 clones: every member at exactly 4, a power of two, + # moved two floats below it, as far as the bound allows there. + (15, 900.0, 15, -2), + # Two floats below 900 at 15 clones: c two floats below 4. + (15, float(np.nextafter(np.nextafter(900.0, 0.0), 0.0)), 15, 1), + # Two floats below 448 at 7 clones: c two floats below 8. + (7, float(np.nextafter(np.nextafter(448.0, 0.0), 0.0)), 8, 1), + # Two floats below 192 at 3 clones: c three floats below 16. + (3, float(np.nextafter(np.nextafter(192.0, 0.0), 0.0)), 4, 1), + ], + ) + def test_beyond_one_clone_members_near_a_power_of_two_are_held_to_two_floats( + self, clone_count: int, source_weight: float, copies: int, moved_floats: int + ) -> None: + # (45 / K, 60 / K] holds a power of two at these clone counts. Every + # member of the family has its halves re-split unevenly, as the #970 + # anchor re-splits a trimmed pair, with its pair sum moved off the + # stage's fl(w / n) and onto a weight within two floats of the power, + # so the hold reaches two floats either side. The stage's weight lies + # within that reach: one float either way is the most the anchor's + # roundings move a pair sum where the floats either side are as wide, + # and below a power of two they are half as wide. The receipt is + # green. One float of reach would leave the stage's weight out of the + # run where c is the power itself. + tool = _load_tool() + scenario = { + "weights": [source_weight, 9_000.0], + "wealth": [7e3, 1.0], + "incomes": [BAND_INCOMES[0], BAND_INCOMES[37_700]], + "top_cells": {0: {250_000: 1_000_000.0}}, + } + split, frame = _clone_first_split_then_cloned(clone_count, scenario) + assert split.copies_created == clone_count * (copies - 1) + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + roots = np.r_[ + np.int64(1), + copy_id_table.mint( + [1] * (clone_count - 1), "geographic_support", range(1, clone_count) + ), + ] + for root in roots: + members = np.r_[ + root, + copy_id_table.mint( + [root] * (copies - 1), "cgt_support_split", range(1, copies) + ), + ] + clones = copy_id_table.mint(members, "cgt_incidence_clone", 1) + stage_weight = 2.0 * weights[ids == root][0] + moved = stage_weight + for _ in range(abs(moved_floats)): + moved = float(np.nextafter(moved, np.inf if moved_floats > 0 else 0.0)) + assert tool._pair_rounding_floats(np.array([moved])).tolist() == [2] + clone_half = float(2.0 ** (np.floor(np.log2(moved)) - 2)) + weights[np.isin(ids, members)] = moved - clone_half + weights[np.isin(ids, clones)] = clone_half + if moved_floats == -2: + least, greatest = tool._floats_dividing_to( + np.array([moved]), np.array([copies]), np.array([1]) + ) + assert not least[0] <= stage_weight * copies <= greatest[0] + + _, problems, reaches = _hold_reaches( + _rebuild(frame, household_weights=weights), scenario, tool + ) + + assert problems == {} + assert reaches == {2: clone_count} - def test_a_household_the_correction_would_divide_keeps_the_receipt_green( + def test_a_household_the_correction_would_divide_is_passed_over( self, monkeypatch ) -> None: # Every household is selected, and the heaviest left at one copy # weighs 60: the correction it could carry would lift it to a weight # that counts two copies (test_uk_cgt_support.py). The split passes - # it over (microcosm#931), so the recompute divides the households - # the stage divided. Had the 60 household carried the correction, the - # recompute would find a family the stage never made. + # it over (microcosm#931), so its weight stays 60 and the recompute + # divides the households the stage divided. Had the 60 household + # carried the correction, it would be a whole household at a weight + # that counts two copies. The family test holds a household to the + # bracket, and that weight lies within a relative 1e-9 of 60, where + # the bracket cannot tell a family of two from a whole household, so + # the receipt would be green as well (microcosm#931 review): only the + # stage's own test sees the check. from microcosm.build.uk_runtime import cgt_support scenario = { @@ -1696,9 +1878,16 @@ def test_a_household_the_correction_would_divide_keeps_the_receipt_green( "incomes": [BAND_INCOMES[0]] * 4, "top_cells": {0: {250_000: 1_000.0}}, } - _, frame = _clone_first_split_then_cloned(1, scenario) + + def stage_weight(split) -> float: + household = split.frame.table("household") + weights = split.frame.weights_for("household").values + return float(weights[household["household_id"].to_numpy() == 3][0]) + + split, frame = _clone_first_split_then_cloned(1, scenario) _, problems = _exhausting_receipt(frame, scenario) assert problems == {} + assert stage_weight(split) == 60.0 correct = cgt_support._exact_support_weights @@ -1706,11 +1895,117 @@ def without_the_check(values, *, target, whole, keeps_the_split=None): return correct(values, target=target, whole=whole) monkeypatch.setattr(cgt_support, "_exact_support_weights", without_the_check) - _, unchecked = _clone_first_split_then_cloned(1, scenario) + unchecked_split, unchecked = _clone_first_split_then_cloned(1, scenario) + monkeypatch.undo() _, problems = _exhausting_receipt(unchecked, scenario) + assert problems == {} + assert stage_weight(unchecked_split) > 60.0 + assert cgt_support.cgt_support_copy_counts( + [stage_weight(unchecked_split)], 60.0, clone_count=1 + ).tolist() == [2] + + def test_a_two_copy_family_that_folds_onto_a_multiple_is_not_reported_missing( + self, + ) -> None: + # A household one float above 60 is divided into two copies, each at + # 30 plus a float. One member's pair sum then moves one float down, + # as the #970 anchor's roundings can move a trimmed pair, and the + # family's fold re-sums to 60, where an exact recount gives one copy: + # the tool before reported the family missing, though the stage made + # it right (microcosm#931 review). The family test now holds it to + # the bracket, which counts two copies at the fold times 1 + 1e-9, so + # it accepts the stored family, and its count of two, and the receipt + # is green; the exact recount still records the lost copy. + scenario = { + "weights": [_ulps_above(60.0, 1), 9_000.0], + "wealth": [7e3, 1.0], + "incomes": [BAND_INCOMES[0], BAND_INCOMES[37_700]], + "top_cells": {0: {250_000: 1_000_000.0}}, + } + split, frame = _clone_first_split_then_cloned(1, scenario) + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + (clone,) = copy_id_table.mint([1], "cgt_incidence_clone", 1) + assert weights[ids == 1][0] + weights[ids == clone][0] == _ulps_above(30.0, 1) + weights[ids == 1] = 20.0 + weights[ids == clone] = 10.0 + moved = _rebuild(frame, household_weights=weights) + assert _load_tool()._pre_split_household_weights(moved)[ids == 1][0] == 60.0 + + receipt, problems = _exhausting_receipt(moved, scenario) + + assert problems == {} + assert receipt["families"] == 1 + assert receipt["copies"] == split.copies_created == 1 + assert receipt["copies_recomputed"] == 0 + + @pytest.mark.parametrize("clone_count", [1, 3]) + def test_a_family_dropped_or_added_away_from_a_multiple_is_reported( + self, monkeypatch, clone_count: int + ) -> None: + # A stage that keeps the 100 household whole and divides the 50 one + # into two copies, in every clone, against the rule's two and one: + # both weights lie far from a multiple of 60, so the bracket holds + # each to the rule's count, and the family test reports one family + # missing and one extra per clone. + from microcosm.build.uk_runtime import cgt_support + + scenario = { + "weights": [100.0, 50.0, 9_000.0], + "wealth": [7e3, 6e3, 1.0], + "incomes": [BAND_INCOMES[0]] * 2 + [BAND_INCOMES[37_700]], + "top_cells": {0: {250_000: 1_000_000.0}}, + } + counted = cgt_support.cgt_support_copy_counts + + def dropped_and_added(weights, maximum_copy_weight, *, clone_count): + counts = counted(weights, maximum_copy_weight, clone_count=clone_count) + source = clone_count * np.asarray(weights, dtype=float) + counts = np.where(np.isclose(source, 100.0), 1, counts) + return np.where(np.isclose(source, 50.0), 2, counts) + + with monkeypatch.context() as patched: + patched.setattr(cgt_support, "cgt_support_copy_counts", dropped_and_added) + _, frame = _clone_first_split_then_cloned(clone_count, scenario) + + receipt, problems = _exhausting_receipt(frame, scenario) + assert problems == { - "support_split_selection_stored": {"missing": 0, "extra": 1} + "support_split_selection_stored": { + "missing": clone_count, + "extra": clone_count, + } } + assert receipt["families"] == clone_count + + def test_a_family_whose_members_weigh_nothing_is_reported_not_refused( + self, + ) -> None: + # Every member of the 900 family carries no weight, which no family + # the split divides does. The hold is asked about every stored + # family, and its members share one pair sum, but it takes only a + # positive normal one, so the family falls to the bracket and the + # family test reports it missing (the rule no longer selects it), + # where a run of weights stepped from zero would reach below zero. + _, frame = _clone_first_split_then_cloned(1, _NESTING_SCENARIO) + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + members = np.r_[ + np.int64(1), copy_id_table.mint([1] * 14, "cgt_support_split", range(1, 15)) + ] + weights[np.isin(ids, members)] = 0.0 + weights[np.isin(ids, copy_id_table.mint(members, "cgt_incidence_clone", 1))] = ( + 0.0 + ) + + _, problems, exact, within = _held_families( + _rebuild(frame, household_weights=weights), _NESTING_SCENARIO + ) + + assert problems == { + "support_split_selection_stored": {"missing": 1, "extra": 0} + } + assert (exact, within) == (5, 0) def test_a_household_the_correction_would_move_the_walk_past_keeps_it_green( self, monkeypatch @@ -1788,13 +2083,14 @@ def test_at_one_clone_three_equal_halves_do_not_vouch_for_a_family( assert (exact, within) == (5, 1) def test_the_hold_counts_every_weight_change_after_the_split(self) -> None: - """The K = 1 hold's premises, read from the spine graph and the + """The count hold's premises, read from the spine graph and the stages' code (microcosm#931 review). ``_SUPPORT_SPLIT_CORRECTED_ROWS`` counts the one-row exact-total corrections that can land on a support family member between the split - and the tool's read, and ``_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS`` how - far the rest can move an uncorrected one. A spine node changes the + and the tool's read, and ``_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS`` (two + floats near a power of two) how far the rest can move an uncorrected + one, at any clone count. A spine node changes the household weights only where it creates or expands rows or declares weights: the executor refuses any other node's weights and frames. After the split those are @@ -1846,36 +2142,63 @@ def test_the_hold_counts_every_weight_change_after_the_split(self) -> None: } assert boundaries == {"uk.identity@1"} - # Each call that corrects a frame's total, by the function making it: - # a call to spi_support's helpers, or to a function of the stage's - # module that wraps them. + # Each call that corrects a frame's total, by the function or method + # making it: a call to spi_support's helpers, or to a function of the + # stage's module that wraps them, by bare name or as an attribute + # (spi_support._importance_weights_with_exact_total(...)). Every + # function, method and nested function counts its own calls, not + # those of the functions it nests. helpers = {"_exact_total_correction", "_importance_weights_with_exact_total"} stages = ("cgt_support_split", *weighing) modules = {graph_kernels._stage_module(stage) for stage in stages} assert modules == {cgt_support, cgt_structure, cgt_residential_split} + defined = ast.FunctionDef | ast.AsyncFunctionDef + nests = defined | ast.ClassDef + + def collect(node, prefix: str, into: dict[str, ast.AST]) -> None: + """Every function and method under ``node``, by qualified name.""" + + for child in ast.iter_child_nodes(node): + if isinstance(child, defined): + into[prefix + child.name] = child + collect(child, f"{prefix}{child.name}.", into) + elif isinstance(child, ast.ClassDef): + collect(child, f"{prefix}{child.name}.", into) + else: + collect(child, prefix, into) + + def called(function) -> list[str]: + """The names ``function`` calls outside the functions it nests.""" + + names: list[str] = [] + pending = list(ast.iter_child_nodes(function)) + while pending: + node = pending.pop() + if isinstance(node, nests): + continue + if isinstance(node, ast.Call): + name = getattr(node.func, "id", getattr(node.func, "attr", None)) + if name is not None: + names.append(name) + pending.extend(ast.iter_child_nodes(node)) + return names + corrections: dict[tuple[str, str], int] = {} for module in modules: tree = ast.parse(inspect.getsource(module)) - functions = { - node.name: node - for node in tree.body - if isinstance(node, ast.FunctionDef) - } - - def called(function) -> list[str]: - return [ - sub.func.id - for sub in ast.walk(function) - if isinstance(sub, ast.Call) and isinstance(sub.func, ast.Name) - ] - + functions: dict[str, ast.AST] = {} + collect(tree, "", functions) + assert { + node.name for node in ast.walk(tree) if isinstance(node, defined) + } == {function.name for function in functions.values()} wrappers = { - name - for name, function in functions.items() - if name.startswith("_exact") and set(called(function)) & helpers + function.name + for function in functions.values() + if function.name.startswith("_exact") + and set(called(function)) & helpers } for name, function in functions.items(): - if name in wrappers: + if function.name in wrappers: continue count = sum(item in helpers | wrappers for item in called(function)) if count: @@ -1906,6 +2229,7 @@ def called(function) -> list[str]: # The anchor scales a clone half by at most one. assert CGT_ANCHOR_MAXIMUM_FACTOR == 1.0 assert tool._SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS == 1 + assert tool._SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS_NEAR_A_POWER_OF_TWO == 2 # The stages after the split that change no weight pass their input's # household weights to every frame they build. diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 5672680fad..bbb7dbb8e3 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -850,28 +850,42 @@ def _e8_support_split( ``K`` the geographic clone count the stage read from the frame (microcosm#931; the recompute reads it from the pre-split table the same way). The fold re-sums the ``n`` copies, which can land ulps from ``w``: - at ``K = 1`` for a weight a few ulps above a multiple of the maximum, or - where an exact-total correction fell on a member, and at ``K > 1`` by - more than the count's snap absorbs. So the count is not recounted from - the fold. At ``K = 1`` a family of more than three members whose clone - pair sums are all one value ``c`` is held exactly: the stored ``n`` must - be the count of a weight ``w`` with ``fl(w / n) == c``, or, where fewer - than four of its members show their clone pair untouched, with - ``fl(w / n)`` within one float of ``c``, the most the #970 anchor and the - residential arms move an uncorrected member - (``_support_count_held_exactly``). Every other family, and every family - at ``K > 1``, is held to the family weights' relative tolerance of 1e-9: - the stored ``n`` must lie between the counts at the folded weight times - ``1 - 1e-9`` and ``1 + 1e-9``. That bracket opens for a family whose - source-scaled weight ``K x w`` lies within a relative 1e-9 of a multiple - of 60 (at ``K = 1``, its own weight), where it accepts ``n`` or - ``n + 1``. A stored count moved there on its own is still reported, by - the flag and family-weight checks, but a family the stage split + for a weight a few ulps above a multiple of the maximum (at ``K > 1``, by + more than the count's snap absorbs), or where an exact-total correction + or the #970 anchor's and the residential arms' roundings moved a member. + So neither a family's count nor whether a household is a family at all + is recounted from the fold. At any ``K`` a family of more than three + members is held exactly where it can be (``_support_count_held_exactly``). + Where at least four of its members show their two clone halves at equal + weights and share one pair sum ``c``, whatever the others carry, the + stored ``n`` must be the count of a weight ``w`` with ``fl(w / n) == c``. + Where every member shares ``c`` but fewer show equal halves, + ``fl(w / n)`` may lie within one float of ``c``, or two where ``c`` sits + that close to a power of two, the most the anchor and the arms move an + uncorrected member. Every other family, and whether each household is a + family at all, is held to the family weights' relative tolerance of + 1e-9: the counts at the folded weight times ``1 - 1e-9`` and + ``1 + 1e-9`` bound the stored ``n``, and a household the rule selects is + possibly a family where the higher count exceeds one and possibly whole + where the lower is one, its stored state accepted in between. Where the + hold applies, its counts decide both instead. That bracket opens for a + household whose source-scaled weight ``K x w`` lies within a relative + 1e-9 of a multiple of 60 (at ``K = 1``, its own weight), where it accepts + ``n`` or ``n + 1`` copies, a whole household or a family of two among + them. A stored count moved there on its own is still reported, by the + flag and family-weight checks, but a household the stage split consistently into one copy more there, every member at ``w / (n + 1)`` - and indexed in full, is reported by no check. Where the exact hold - applies, that blind spot narrows to at most the multiple itself (the - multiple and the float either side of it where ``c`` may have moved). - The receipt's ``copies_recomputed`` stays the exact recount from the + and indexed in full, is reported by no check. Where the hold applies, + that blind spot narrows to a few floats of ``w`` next to the count's + step, for some counts ``n``. At ``K = 1`` the step sits a float above the + multiple, so at most the multiple itself is blind where the family is + held exactly, and the multiple and the float either side of it in the + one-float tier. At ``K > 1`` the snapped count's step sits 3 to 17 + floats above the multiple of ``60 / K`` (at ``K`` from 2 to 15): at most + the float on either side of the step is blind where the family is held + exactly, and those up to three floats from it in the one-float tier, so a + family on the multiple itself split into one copy more is reported. The + receipt's ``copies_recomputed`` stays the exact recount from the re-summed fold, so it can differ from ``copies`` by one for each family whose re-summed weight lands across a multiple, or that was split into one copy more, while the count check passes. @@ -993,7 +1007,7 @@ def _e8_support_split( receipt["families"] = int(len(family_positions)) # Fold: clones onto originals (pair sums), then copies onto roots. Which - # members' halves agree feeds the K = 1 count hold below. + # members' halves agree feeds the count hold below. pre_clone, equal_halves = _pre_clone_fold(frame) folded = pre_clone.copy() np.add.at(folded, root_positions, pre_clone[lineage.copy_positions]) @@ -1064,53 +1078,58 @@ def recompute(person_t, household_t, weights_t): receipt["households_selected"] = int(original["selected"].sum()) receipt["copies_recomputed"] = int((original["copies"] - 1).sum()) - recomputed_family = original["selected"] & (original["copies"] > 1) - stored_family = pd.Series(stored_copies[pre_split] > 1, index=pre_ids) - stored_count = pd.Series(stored_copies[pre_split], index=pre_ids) - missing_families = int((stored_family & ~recomputed_family).sum()) - extra_families = int((recomputed_family & ~stored_family).sum()) - if missing_families or extra_families: - problems["support_split_selection_stored"] = { - "missing": missing_families, - "extra": extra_families, - } - agreed = (recomputed_family & stored_family).to_numpy() - stored_agreed = stored_count.to_numpy()[agreed] - # The fold re-sums the n copies, which can land ulps from w: at K = 1 for - # a weight a few ulps above a multiple of 60, or where an exact-total - # correction fell on a member, and at K > 1 by more than the count's - # K-ulp snap absorbs (an exact multiple of 60 at K = 11, for one). An - # exact recount of the fold would flag a family the stage counted right, - # so the count is held to the family weights' tolerance: the stored n - # must lie between the counts at the folded weight times 1 - 1e-9 and - # 1 + 1e-9 (microcosm#931 review). That bracket opens for a family whose - # source-scaled weight K x w lies within a relative 1e-9 of a multiple of - # 60 (at K = 1, its own weight), where it cannot tell n from n + 1. - agreed_weights = pre_weights[agreed] + # The fold re-sums the n copies, which can land ulps from w: for a weight + # a few ulps above a multiple of 60 (at K > 1, by more than the count's + # K-ulp snap absorbs: an exact multiple of 60 at K = 11, for one), or + # where an exact-total correction or the anchor's and the arms' + # roundings moved a member. An exact recount of the fold would report a + # count the stage got right, or a family missing that it made (a + # two-copy family a float above 60 that folds onto 60, for one), so + # neither is recounted exactly (microcosm#931 review). Each household is + # held to the family weights' tolerance, the counts at its folded weight + # times 1 - 1e-9 and 1 + 1e-9, at the K-aware count: one the rule selects + # is possibly a family where the higher count exceeds one and possibly + # whole where the lower is one, its stored state accepted in between, + # one it does not select is whole, and a family's stored n must lie + # between the two counts. That bracket opens for a household whose + # source-scaled weight K x w lies within a relative 1e-9 of a multiple + # of 60 (at K = 1, its own weight), where it cannot tell n copies from + # n + 1. A family of more than three members whose members carry its + # weight as _support_count_held_exactly requires is held exactly + # instead, at any K: the counts at the ends of its run of weights decide + # both its count and that it is a family. + selected = original["selected"].to_numpy(dtype=bool) + stored_count = stored_copies[pre_split] + stored_family = stored_count > 1 lowest, highest = ( cgt_support_copy_counts( - agreed_weights * factor, + pre_weights * factor, CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=clone_count, ) for factor in (1.0 - 1e-9, 1.0 + 1e-9) ) - if clone_count == 1: - # At K = 1 a family of more than three members that all fold to one - # weight c is held exactly instead: the stored n must be the count - # of a weight w that divides by n to c, or to within the one float - # the later stages' roundings can move c where fewer than four - # members show their pair untouched (_support_count_held_exactly). - held, held_lowest, held_highest = _support_count_held_exactly( - pre_clone, - equal_halves, - roots=np.flatnonzero(pre_split)[agreed], - stored_counts=stored_agreed, - lineage=lineage, - ) - lowest = np.where(held, held_lowest, lowest) - highest = np.where(held, held_highest, highest) - count_off = int(((stored_agreed < lowest) | (stored_agreed > highest)).sum()) + held, held_lowest, held_highest = _support_count_held_exactly( + pre_clone, + equal_halves, + roots=np.flatnonzero(pre_split)[stored_family], + stored_counts=stored_count[stored_family], + lineage=lineage, + clone_count=clone_count, + ) + lowest[stored_family] = np.where(held, held_lowest, lowest[stored_family]) + highest[stored_family] = np.where(held, held_highest, highest[stored_family]) + may_be_family = selected & (highest > 1) + may_be_whole = ~selected | (lowest == 1) + missing_families = int((stored_family & ~may_be_family).sum()) + extra_families = int((~stored_family & ~may_be_whole).sum()) + if missing_families or extra_families: + problems["support_split_selection_stored"] = { + "missing": missing_families, + "extra": extra_families, + } + agreed = stored_family & may_be_family + count_off = int(((stored_count < lowest) | (stored_count > highest))[agreed].sum()) if count_off: problems["support_split_copies_stored"] = count_off @@ -1136,13 +1155,17 @@ def recompute(person_t, household_t, weights_t): #: it does spi_support's shared fallback put it on a root or copy), the #: incidence clone's and the residential split's. The #970 anchor applies #: none. So at most this many support-family members carry a corrected -#: weight, and a family of more members has one that carries none. -#: test_uk_identity_stability_receipts derives both counts' premises from -#: the spine graph and the stages' code. +#: weight, and among more members than this one carries none. +#: test_uk_identity_stability_receipts derives the premises of this count +#: and of the two reaches below from the spine graph and the stages' code. _SUPPORT_SPLIT_CORRECTED_ROWS = 3 #: How many floats the #970 anchor and the residential arms' fold can move -#: the pair sum of a K = 1 family member no correction touched. +#: the pair sum of a support-family member no correction touched, where the +#: floats either side of it are equally wide. _SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS = 1 +#: The same where that pair sum sits within this many floats of a power of +#: two, below which the floats are half as wide. +_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS_NEAR_A_POWER_OF_TWO = 2 def _support_count_held_exactly( @@ -1152,48 +1175,78 @@ def _support_count_held_exactly( roots: np.ndarray, stored_counts: np.ndarray, lineage: _SupportCopyLineage, + clone_count: int, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """The K = 1 support families whose copy count is held exactly, and its counts. - - ``roots`` are the positions of the families the recompute and the stored - layer agree on, ``stored_counts`` their stored counts ``n``. A family is - held where all ``n`` members are present (its root and copies 1 to - ``n - 1``), ``n`` exceeds :data:`_SUPPORT_SPLIT_CORRECTED_ROWS`, and every - member's clone pair sum (``pre_clone``) is one value ``c``. The split gave - every member ``fl(w / n)``, ``w`` the family's source weight, and at most - that many members carry a weight an exact-total correction moved after - it, so at least one carries ``fl(w / n)`` as only the later stages' - roundings can have moved it: - - - Where more than that many members show equal halves - (``equal_halves``), one of those is uncorrected, and its pair sum is - exactly ``fl(w / n)``. The incidence clone halves a weight exactly. The - #970 anchor scales a pair's clone half by a factor of at most one and - adds what it took to the original. The residential split divides only - a liable clone half, which the anchor leaves at factor one, and never - an original (the clone stage gives gains to clone carriers alone). So - an uncorrected member whose halves are still equal after the anchor - and the arms' fold carries both at exactly half of ``fl(w / n)``, and - ``c == fl(w / n)``. - - Otherwise ``c`` is within :data:`_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS` - float of ``fl(w / n)``. At ``K = 1`` a member's weight lies in [32, - 64) once ``n > 3``. The anchor's three roundings move a trimmed pair's - sum by at most three quarters of an ulp before its last rounding. The - anchor refuses gains on any clone member but its carrier, so a liable - clone half becomes its own row and one arm, and the two products and - their sum keep it within one ulp of itself, half an ulp of the pair. - - A held family's stored ``n`` must then be the count - ``cgt_support_copy_counts(w', 60, clone_count=1)`` of some ``w'`` whose - quotient ``fl(w' / n)`` lies that close to ``c``. Those ``w'`` are one - run of floats, and the count rises with ``w'``, so the counts at the - run's ends bound it. A family the stage counted right always passes. One - it split consistently into one copy more or fewer passes only where that - run reaches across the count's step. By design arithmetic over every - ``n`` from 4 to 9,998, that leaves at most the multiple of 60 itself - where the family is held exactly, and the multiple and the float either - side of it where the hold reaches one float. Returns the held mask over - ``roots`` and the counts at the run's ends (zero where not held). + """The support families whose copy count is held exactly, and its counts. + + ``roots`` are the positions of the stored families, ``stored_counts`` + their stored counts ``n``. The split counted ``n`` on the family's + weight ``w`` as ``cgt_support_copy_counts(w, 60, clone_count=K)``, ``K`` + the frame's geographic clone count (``clone_count``), and gave every + member ``c = fl(w / n)``. A family is held where all ``n`` members are + present (its root and copies 1 to ``n - 1``), ``n`` exceeds + :data:`_SUPPORT_SPLIT_CORRECTED_ROWS`, the most members the exact-total + corrections after the split can have moved, and its members' clone pair + sums (``pre_clone``) pin ``c`` in one of two ways: + + - Exactly, where more than that many members show their two clone halves + at equal weights (``equal_halves``) and those members' pair sums are + all one value, whatever the other members carry (trimmed or liable + members, a corrected one). One of those members is uncorrected. The + incidence clone halves a weight exactly. The #970 anchor scales a + pair's clone half by a factor of at most one and adds what it took to + the original. The residential split divides only a liable clone half, + which the anchor leaves at factor one, and never an original (the + clone stage gives gains to clone carriers alone). So an uncorrected + member whose halves are still equal after the anchor and the arms' + fold carries both at exactly ``c / 2``, and the shared value is ``c``. + - To within a float or two, where every member's pair sum is one value + ``c'`` but fewer than that many show equal halves: at least one member + is uncorrected, and ``c`` lies within + :data:`_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS` float of ``c'``, or + :data:`_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS_NEAR_A_POWER_OF_TWO` where + ``c'`` sits that close to a power of two (:func:`_pair_rounding_floats`). + Let ``s`` be the spacing of the floats at and just above ``c``, so its + half ``c / 2`` has spacing ``s / 2``. The anchor sets a trimmed pair's + clone half to ``k = fl(c / 2 x f)`` and its original to + ``fl(c / 2 + fl(c / 2 - k))``. The difference is exact where ``k`` is + at least ``c / 4`` and otherwise rounds by at most ``s / 4``, and the + sum rounds by at most ``s / 2``, so the pair sum lies within + ``3 s / 4`` of ``c`` before its own rounding. The anchor refuses gains + on a clone member but its carrier, so a liable clone half becomes its + own row and one arm. The two arms' products and their fold land within + a float of the half, so the pair sum lies within ``s / 2`` of ``c`` + before its rounding. Where the floats either side of ``c`` are ``s`` + apart, both round to within one float of ``c``. Below a power of two + the floats are half as wide, so the bound allows two there. At + ``K = 1``, ``c`` lies in (45, 60] once ``n > 3``, far from a power of + two. At ``K > 1`` it lies in (45 / K, 60 / K] (to within the snap), + which holds one at some ``K``: 16 at ``K = 3``, 8 at 6 and 7, and 4 at + 12 to 15. + + Every other family keeps the bracket. In a family of three or fewer + members every member may carry a corrected weight. In one whose members + differ, with fewer than four equal halves to vouch for ``c``, no member + is known to be uncorrected. A held family's stored ``n`` must then be + the count ``cgt_support_copy_counts(w', 60, clone_count=K)`` of some + ``w'`` whose quotient ``fl(w' / n)`` lies that close to the shared + value. Those ``w'`` are one run of floats, which holds ``w``. The count + rises with ``w'``, at ``K > 1`` too, since the snap moves a quotient + within ``K`` ulps of an integer onto that integer and so keeps their + order. So the counts at the run's ends bound it, and a family the stage + counted right always passes. One it split consistently into one copy + more or fewer passes only where that run reaches across the count's + step. By design arithmetic over every ``n`` from 4 to 9,998: at + ``K = 1``, where the step sits a float above the multiple of 60, that + leaves at most the multiple itself blind where the family is held + exactly (for 31 of those counts), and the multiple and the float either + side of it in the one-float tier. At ``K`` from 2 to 15, where the snap + puts the step 3 to 17 floats above the multiple of ``60 / K``, it leaves + at most the float on either side of the step where the family is held + exactly, and up to three floats from it in the one-float tier, so a + family on the multiple itself split into one copy more is reported. + Returns the held mask over ``roots`` and the counts at the run's ends + (zero where not held). """ from microcosm.build.uk_runtime.cgt_support import ( @@ -1202,44 +1255,77 @@ def _support_count_held_exactly( ) rows = len(pre_clone) + roots = np.asarray(roots, dtype=np.int64) copy_roots = lineage.root_positions - copy_positions = lineage.copy_positions + members = np.concatenate([roots, lineage.copy_positions]) + member_roots = np.concatenate([roots, copy_roots]) + sums = pre_clone[members] + equal = equal_halves[members] + + def spread(mask: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """The least and greatest pair sum of each family's members in ``mask``.""" + + least = np.full(rows, np.inf) + greatest = np.full(rows, -np.inf) + np.minimum.at(least, member_roots[mask], sums[mask]) + np.maximum.at(greatest, member_roots[mask], sums[mask]) + return least[roots], greatest[roots] + copies_present = np.bincount(copy_roots, minlength=rows)[roots] - differing = np.bincount( - copy_roots, - weights=(pre_clone[copy_positions] != pre_clone[copy_roots]).astype(float), - minlength=rows, - )[roots] - untouched = equal_halves[roots].astype(np.int64) + np.bincount( - copy_roots, - weights=equal_halves[copy_positions].astype(float), - minlength=rows, - )[roots].astype(np.int64) - held = ( - (copies_present + 1 == stored_counts) - & (stored_counts > _SUPPORT_SPLIT_CORRECTED_ROWS) - & (differing == 0) - ) + showing_equal_halves = np.bincount(member_roots[equal], minlength=rows)[roots] + least, greatest = spread(np.ones(len(members), dtype=bool)) + least_equal, greatest_equal = spread(equal) + complete = (copies_present + 1 == stored_counts) & ( + stored_counts > _SUPPORT_SPLIT_CORRECTED_ROWS + ) + on_equal_halves = ( + complete + & (showing_equal_halves > _SUPPORT_SPLIT_CORRECTED_ROWS) + & (least_equal == greatest_equal) + ) + on_every_member = complete & ~on_equal_halves & (least == greatest) + shared = np.where(on_equal_halves, least_equal, least) + # A pair sum below the normal floats would not halve exactly; no family + # the split divided carries one. + held = (on_equal_halves | on_every_member) & (shared >= np.finfo(float).tiny) lowest = np.zeros(len(roots), dtype=np.int64) highest = np.zeros(len(roots), dtype=np.int64) if held.any(): - floats = np.where( - untouched[held] > _SUPPORT_SPLIT_CORRECTED_ROWS, - 0, - _SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS, - ) - least, greatest = _floats_dividing_to( - pre_clone[roots[held]], stored_counts[held], floats + floats = np.where(on_equal_halves[held], 0, _pair_rounding_floats(shared[held])) + least_weight, greatest_weight = _floats_dividing_to( + shared[held], stored_counts[held], floats ) lowest[held] = cgt_support_copy_counts( - least, CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=1 + least_weight, CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=clone_count ) highest[held] = cgt_support_copy_counts( - greatest, CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=1 + greatest_weight, CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=clone_count ) return held, lowest, highest +def _pair_rounding_floats(weights: np.ndarray) -> np.ndarray: + """The hold's reach, in floats, around each family's shared pair sum. + + :data:`_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS`, or + :data:`_SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS_NEAR_A_POWER_OF_TWO` where + the weight sits within that many floats of a power of two. Positive + floats order as their bit patterns do, so the floats between a weight + and a power of two are the difference of the two read as integers. + """ + + values = np.asarray(weights, dtype=float) + _, exponent = np.frexp(values) + power_below = np.ldexp(1.0, exponent - 1) + power_above = np.ldexp(1.0, exponent) + apart = np.minimum( + values.view(np.int64) - power_below.view(np.int64), + power_above.view(np.int64) - values.view(np.int64), + ) + near = _SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS_NEAR_A_POWER_OF_TWO + return np.where(apart <= near, near, _SUPPORT_SPLIT_PAIR_ROUNDING_FLOATS) + + def _floats_dividing_to( quotients: np.ndarray, divisors: np.ndarray, floats: np.ndarray ) -> tuple[np.ndarray, np.ndarray]: @@ -1987,20 +2073,28 @@ def plans_equal(left: pd.Series, right: pd.Series) -> bool: "pair sum against w / n at rtol 1e-9 / atol 1e-6 (the fold and " "the anchor each cost one float rounding generation); " "support-split mass record: new_total against old_total at rtol " - "1e-9; support-split copy counts: at K = 1, for a family of more " - "than three members whose clone pair sums are all one weight c, " - "exactly the count of a weight w with fl(w / n) == c, or, where " - "fewer than four members show their clone pair untouched, with " - "fl(w / n) within one float of c, a reach that can take in the " - "multiple of 60 itself and then does not report a family split " - "consistently into one copy more; for every other family and at " - "K > 1, between the counts at the folded family weight times " - "1 - 1e-9 and 1 + 1e-9, which for a family whose source-scaled " - "weight K x w lies within a relative 1e-9 of a multiple of 60 (at " - "K = 1, its own weight) does not report a family split " - "consistently into one copy more; copies_recomputed is the exact " - "recount from the re-summed fold; support-split " - "families, flags and student_loan_plan: exact equality; " + "1e-9; support-split copy counts, at any K: for a family of more " + "than three members, exactly the count of a weight w with " + "fl(w / n) == c where at least four members show their two clone " + "halves at equal weights and share the pair sum c, whatever the " + "others carry, or, where every member's pair sum is c but fewer " + "show equal halves, with fl(w / n) within one float of c, or two " + "where c sits within two floats of a power of two; either run " + "can reach across the count's step from the floats next to it " + "(at K = 1, the multiple of 60 itself), where it does not report " + "a family split consistently into one copy more or fewer; for " + "every other family, between the counts at the folded family " + "weight times 1 - 1e-9 and 1 + 1e-9, which for a household whose " + "source-scaled weight K x w lies within a relative 1e-9 of a " + "multiple of 60 (at K = 1, its own weight) does not report one " + "split consistently into one copy more; copies_recomputed is the " + "exact recount from the re-summed fold; support-split families: " + "a household the rule selects is possibly a family where its " + "count at the folded weight times 1 + 1e-9 exceeds one and " + "possibly whole where its count at the folded weight times " + "1 - 1e-9 is one, its stored state accepted in between, or the " + "held counts where the count is held; support-split flags and " + "student_loan_plan: exact equality; " "band-donor seats, bands and " "carriers: exact equality; band-donor weights within a band and " "against the rerun: rtol 1e-9 / atol 1e-6; band-donor mass " @@ -2353,7 +2447,7 @@ def _pre_clone_fold(frame) -> tuple[np.ndarray, np.ndarray]: The mask is true on an original whose clone carries bitwise the original's own weight, with the residential arms folded onto both, and on every row of an artifact without the clone layer; false on the - clones and the arms. The K = 1 count hold of ``_e8_support_split`` reads + clones and the arms. The count hold of ``_e8_support_split`` reads it: the incidence clone halves each weight exactly, and halves still equal after the #970 anchor and the arms' fold are both still exactly half the weight the clone halved, unless an exact-total correction moved From a4bb4a2af66e658b5bcad34c6a9870964a1ff68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:18:59 +0000 Subject: [PATCH 036/108] Bind what the injected engine computes through into the engine stages (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María's rulings of 9 Nov (5, 7, 8 and 9) asked for four things: - bind the first-party modules the injected engine computes through into the engine stages, through a second walk root at the engine class; - have the walk see the names microcosm.graph exports lazily, with the create kernel's H2-only use of load_source listed as a stop; - keep SeedSource.NONE for every spine stage kernel; - keep the graph_frames move. This commit does all four and the re-review's changelog nit. It changes keys only. - The engine root. test_uk_graph roots a second walk at PolicyEngineUKEngine, the class that graph_kernels' hermetic H2 builder and the spine build both construct (checked). - The walk follows the engine methods each engine stage's code names, by attribute on any receiver or through getattr. It also follows the constructor and every method they call on self, at the first-party scope with the stage rule's tables, and _stage_walks adds it to the stage's own walk. - The 16 engine stages call materialize and variable_metadata, and private_school_attendance also calls parameters and simulation_batches through getattr. - Beyond the three hand-bound engine modules, the walk reaches one module: microcosm.frame.materialize, through which the adapter builds each simulation's dataset. graph_kernels gains _ENGINE_COMPUTE_MODULES = (frame_materialize,) beside _ENGINE_MODULES, and every engine stage binds both (_ENGINE_STAGE_MODULES). - A new test derives the group exactly. The engine-modules test keeps _ENGINE_MODULES exact. Another stage may bind a module of the group only where its own transform computes through it, as the SPI spine's three stages and the band donors do. - The engine reaches the concept vocabulary (policyengine_uk_concepts, concept_mapping and concepts) and frame.rules only through methods no engine stage calls (concept_mapping and export_contract). So they are not bound, and the engine stages do not re-key on edits to the concept vocabulary, unlike the consequence stated with the ruling. - The fixture resolver stop's reason no longer names the engine as held. SOURCE_HOUSEHOLD_ID_COLUMN's listing is unchanged. - Lazy exports. - _resolve now follows a name that a module exports through its module __getattr__ to the submodule that defines it. It reads the table that __getattr__ reads (_LAZY_EXPORT_TABLES: microcosm.graph's _RUNTIME_EXPORTS). - The walk now reads type aliases as definitions. Three of those names are type aliases. - A test checks that every lazy name resolves to the object the package returns. It also checks that every first-party module with a module __getattr__ is listed; us_runtime.sources, which only raises, is listed as exporting nothing. - Resolving the lazy names changes no stage's walk. In the UK kernels, two things surface: - six definitions of microcosm.graph.kernel (the capabilities record, three of its enums, and the context and result records), now listed as core names with reasons; - the create kernel's load_source, now a listed stop. - The stop's reason: the create kernel calls load_source only where no root transform is bound, the hermetic registry's data-only path over the H2 fixture's captured root tables. The spine build installs the root's transform, and H2 parity binds it too (checked). - One test checks that the call sits only in the "self.transform is None" branch. Another checks that no spine kernel and no national-line kernel hashes microcosm.graph.codecs. - The blind-spot paragraph is gone from the kernel test's docstring. - SeedSource. The comment by the NONE assertion records the ruling and why NONE holds: regional_home_values' stream is a stage parameter, which its contract hash covers, and it reads no generator the executor seeds. - The changelog fragment. - The What moves sentence on local_doctrine and target_weights is reworded so that each clause is true. It was checked against the recorded source_hash calls and the kernels' walks. - The fragment now describes the engine root, the lazy exports, the load_source stop and the SeedSource ruling. What moves: - Node keys only. No value, declaration or generated file moves. - graph_kernels.py's bytes changed. So the implementation hashes of the create, identity and claim kernels and of all 53 stage and expand kernels move: 56 of the 99 UK kernels. The 16 engine stages now also hash frame.materialize. - Every node re-keys, by its own hash or by chaining: - the hermetic spine, 79 of 79; - the spine with its gates, 81 of 81; - the full graph with its gates, 111 of 111. - The other 43 kernels keep their hashes. So the national line, whose root is the bound spine checkpoint, keeps its keys, as do the transport graphs' atomic geography nodes. The country spec's and the solve's fingerprints do not move. - uk_spine.json holds only the 54 stages' stage_contract_sha256 values, and the H2 fixture holds no code hash. The fixture regenerates byte-identically, and source_stages.json and the coverage manifest are current. Tests: all engine-free, synthetic or code facts, except H2. - test_uk_graph (50 tests) and 15 related files pass (445 tests). - Ten mutations on a scratch copy each fail the tests that should catch them: - materialize newly reads the concept mapping; - frame.materialize leaves the group; - a stage calls the engine's concept_mapping through getattr; - the walk ignores lazy exports; - load_source moves out of the unbound branch; - a kernel newly reads a lazy export it does not hash; - a stage newly reads a lazy export it does not hash; - a non-engine stage binds frame.materialize without computing through it; - a spine stage hashes the codecs; - a module __getattr__ is not listed. - Ruff check and format are clean on the two edited Python files. - Through the engine slot, the H2 fixture check regenerates byte-identically and test_h2_uk_spine_parity passes. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-helper-bindings.changed.md | 26 +- .../build/uk_runtime/graph_kernels.py | 57 +- .../tests/engine_free/uk/test_uk_graph.py | 633 ++++++++++++++++-- 3 files changed, 626 insertions(+), 90 deletions(-) diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 356400639c..245479ca6a 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,26 +1,27 @@ -Every UK spine stage now hashes each first-party module its transform computes through, and every other UK graph kernel, except nine held for a ruling, hashes each one its own code computes through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. +Every UK spine stage now hashes each first-party module its transform computes through, and each stage that calls the engine also each one the injected engine computes through for it. Every other UK graph kernel, except nine held for a ruling, hashes each one its own code computes through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. -- **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports and module attributes. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: +- **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage, and for a stage that calls the engine also at the engine the builds inject (see The engine), and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports, module attributes and `type` aliases, which read as assignments of their values. So do the names `microcosm.graph` exports lazily through its module `__getattr__`: the walk resolves each to the runtime module that defines it, from the table that `__getattr__` reads, and a test checks that every first-party module with a module `__getattr__` is listed. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: - six listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels); - the stage's own declaration (`country_spec`, `source_manifest` and the spec engine), whose content the node key already hashes as the stage contract; - the listed core modules every node runs through (see Scope); - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; two such names are listed. - A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies it to the UK's other graph kernels (see Kernels). Two more cover the transforms the production spine build installs itself; a transform installed inside the source adapter or the rung sampler must come from its stage's module or be a listed adapter, since neither hands the node what the wrapped transform declares. Control tests pin each way of naming a helper that the rule follows, each kind it skips, the core-module stop, and each way a refusal-only check could write. -- **Scope.** The rule follows every first-party package: `microcosm-build`, `-calibrate`, `-data`, `-diagnostics`, `-fit`, `-frame` and `-graph`, all under the `microcosm.` namespace. What the stages compute through outside `microcosm.build` is now bound as it is inside it, 71 bindings in 47 stages: + A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies it to the UK's other graph kernels (see Kernels). Two more cover the transforms the production spine build installs itself; a transform installed inside the source adapter or the rung sampler must come from its stage's module or be a listed adapter, since neither hands the node what the wrapped transform declares. Control tests pin each way of naming a helper that the rule follows, a lazy export among them, each kind it skips, the core-module stop, each way a refusal-only check could write, and how the engine's second root picks and follows the methods a stage calls. +- **Scope.** The rule follows every first-party package: `microcosm-build`, `-calibrate`, `-data`, `-diagnostics`, `-fit`, `-frame` and `-graph`, all under the `microcosm.` namespace. What the stages compute through outside `microcosm.build` is now bound as it is inside it: 71 bindings in 47 stages, found by the walks from their transforms, and 16 more in the engine stages, found by the walk from their engine (see The engine): - `microcosm.fit.qrf` and `fit.model` in 11 stages: the nine that fit through the regime-gated QRF, and `spi_employment_shell` and `aps_occupation`, whose classifier resolves its fit weights through `fit.model` and whose held-out scores and folds use the QRF's; - - `microcosm.frame.materialize`, whose `engine_tables` copies a frame's tables, in the three stages of `spi_spine` and the band donors; + - `microcosm.frame.materialize`, whose `engine_tables` copies a frame's tables, in the three stages of `spi_spine` and the band donors, and in the 16 engine stages, whose engine builds each simulation's dataset through it; - `microcosm.frame.rules` in `uc_take_up` and `pension_credit_take_up`, which type their engine by the adapter's `ProjectingRulesEngine`, a protocol that extends its `RulesEngine`; - the engine adapter in every spine stage, beside `national_frame`, which stamps each frame it builds with the adapter's ISA income contract and savings income projection; 38 stages that call no engine gain it; - `microcosm.graph.randomness` and its canonical encoder `microcosm.graph.canonical` in `regional_home_values`, which draws its keyed uniforms through them; - `microcosm.calibrate.geography_constants`, the UK region tier, in `hmrc_cgt_gains_spine`, `isa_subscriptions` and `spi_housing_shell`. Listed instead, each with a reason a test checks: - - the core modules every node runs through, where the rule stops, since binding one would re-key every node on any edit to it. They are the Frame's three modules (`microcosm.frame.bundle`, `schema` and `weights`), which a stage's transform meets all three or none of; the graph executor's (`microcosm.graph.executor` and `population`); and the modules that compute node keys (`keys`, `decl` and `kernel`), which no stage's transform meets. No spine key hashes them. H2 cannot catch an edit to the Frame, since the graph's spine and the legacy stage plan both run through it, and H1, whose fixtures run four wrapped kernels through `run_graph` against stored bytes, catches one only on those kernels' frames; an edit that changes a UK value is otherwise left to the Frame's own tests, and a test pins that `Weights.total`, which the UK exact-total corrections target, is NumPy's sum of the weights. One to the executor's modules that changes a value on H2's fixture fails H2, which compares the graph's spine with the legacy stage plan's. One that changes what `node_key` or `source_hash` returns changes every key. A test checks that the walk from `run_graph` meets each of them. `microcosm.graph.canonical` also encodes every key, but `regional_home_values` draws through it, so it is bound there instead. That stage draws through `microcosm.graph.randomness`, whose contract asks for `SeedSource.KEYED`, while every spine stage kernel declares `NONE`; which it should declare is held for a ruling; + - the core modules every node runs through, where the rule stops, since binding one would re-key every node on any edit to it. They are the Frame's three modules (`microcosm.frame.bundle`, `schema` and `weights`), which a stage's transform meets all three or none of; the graph executor's (`microcosm.graph.executor` and `population`); and the modules that compute node keys (`keys`, `decl` and `kernel`), which no stage's transform meets. No spine key hashes them. H2 cannot catch an edit to the Frame, since the graph's spine and the legacy stage plan both run through it, and H1, whose fixtures run four wrapped kernels through `run_graph` against stored bytes, catches one only on those kernels' frames; an edit that changes a UK value is otherwise left to the Frame's own tests, and a test pins that `Weights.total`, which the UK exact-total corrections target, is NumPy's sum of the weights. One to the executor's modules that changes a value on H2's fixture fails H2, which compares the graph's spine with the legacy stage plan's. One that changes what `node_key` or `source_hash` returns changes every key. A test checks that the walk from `run_graph` meets each of them. `microcosm.graph.canonical` also encodes every key, but `regional_home_values` draws through it, so it is bound there instead. That stage draws through `microcosm.graph.randomness`, whose contract asks for `SeedSource.KEYED`. Every spine stage kernel, that stage's included, keeps `SeedSource.NONE`, as María ruled on 9 Nov: its stream is a stage parameter, which its contract hash covers, and it reads no generator the executor seeds; - `microcosm.frame.rules`' `assert_rules_engine_country`, the fourth refusal-only check, which every stage that takes an engine calls first; - `SOURCE_HOUSEHOLD_ID_COLUMN` from `microcosm.frame.concepts`, CopyLineage's default source column, the second listed name. The support channel writes the column by its literal name. Every stage that reads the constant declares the column by that name, which its key hashes, and counts before the copies only beside a copy column, or declares neither, so a renamed constant leaves the lineage readers to refuse or count as before. - Still outside the rule: calls through an object whose class the code never names. The engine the engine stages call is one: the hand-bound engine modules cover the adapter, the engine-block machinery and the UK declaration, but not the adapter's own imports (`policyengine_uk_concepts`, `concept_mapping`, `concepts`, `materialize` and `rules`). + Still outside the rule: calls through any other object whose class the code never names, a name looked up by a computed string, and data files read by path. +- **The engine.** The 16 stages that call the engine compute through the `PolicyEngineUKEngine` the builds inject, whose class no stage's transform names. As María ruled on 9 Nov, a second walk is rooted at that class, the one `graph_kernels`' hermetic H2 builder and the spine build both construct (checked). It follows the methods a stage's code names on the engine, by attribute on any receiver or through `getattr`, with the constructor and every method they call on `self`, at the same scope and with the same tables, and the stage rule's tests hold what it reaches as part of the stage's walk. It matches methods by name, as the engine-call scan does, so it can only over-count. The engine stages call `materialize`, `variable_metadata` and, through `getattr` in `private_school_attendance`, `parameters` and `simulation_batches`. The walk reaches the three hand-bound engine modules and one more, `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. That module is the new `_ENGINE_COMPUTE_MODULES` group in `graph_kernels`, which every engine stage binds and which a test derives exactly. It sits beside `_ENGINE_MODULES`, which no other stage binds, because the SPI spine's three stages and the band donors bind `frame.materialize` for their own tables; any other stage binds it only where its own transform computes through it (checked). The engine reaches the concept vocabulary (`policyengine_uk_concepts`, `concept_mapping` and `concepts`) and `frame.rules` only through methods no engine stage calls (`concept_mapping` and `export_contract`), so none is bound for the engine. No spine stage hashes the concept vocabulary, and only `uc_take_up` and `pension_credit_take_up` hash `frame.rules`, for the protocol they type their engine by. The hermetic fixture's transform resolver no longer names the engine as held. - **What is bound.** `national_frame` joins the modules every spine stage hashes. The stage helper table gains 231 bindings of 54 modules in 49 stages. Among them: - in the CGT and high-value stages: `property_localisation`, `cgt_structure`, `spi_support`'s exact-total helpers, `cgt_imputation`, `cgt_asset_type` and `advani_summers`; - `employer_pension_contributions` in `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`; @@ -34,12 +35,13 @@ Every UK spine stage now hashes each first-party module its transform computes t The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input`, `gates` and `spi_support`, whose replacement strata the sample reads. - **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. Through `graph_implementation_dependencies`, which the root node hashes, it declares `national_sampling` at every rung, since it resets the root's indexes at f100 too, and `frame_sampling` only below f100, where it samples. Its behaviour is unchanged. -- **Kernels.** A second test applies the rule to every kernel the UK registries register but the nine held below: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; and the national gates and readback. It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at three listed kernel stops, each with a reason a test checks: - - the hermetic fixture's transform resolver, since the stage rule holds each stage's transform, but for the engine it injects into the engine stages, whose own imports are outside the rule (see Scope); +- **Kernels.** A second test applies the rule to every kernel the UK registries register but the nine held below: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; and the national gates and readback. It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at four listed kernel stops, each with a reason a test checks: + - the hermetic fixture's transform resolver, since the stage rule holds each stage's transform and, through its second root, the engine the resolver's builder injects into the engine stages (see The engine); - `_minimal_frame`'s rebuild of the SPI support allocation's mass record, which only `hmrc_spi_income_spine` takes, and whose reach that stage's node hashes; - - `UK_GATE_REGISTRY`, which the bound spine reaches only for its checkpoint's gate digests. The national gates evaluate through it and are held there, in a table of their own (see Held for a ruling). + - `UK_GATE_REGISTRY`, which the bound spine reaches only for its checkpoint's gate digests. The national gates evaluate through it and are held there, in a table of their own (see Held for a ruling); + - `microcosm.graph.codecs`' `load_source`, which the walk now sees through `microcosm.graph`'s lazy exports. The create kernel calls it only where no root transform is bound, the hermetic registry's data-only path, which loads the H2 fixture's captured root tables. The spine build installs the root's transform and H2 parity binds it too, so neither takes that path (checked). Binding the codecs, and the content store they name, would re-key the spine root, and with it every spine node and every node downstream of the spine, on edits to code no build runs. As María ruled on 9 Nov, it is a stop instead, and a test checks that no spine kernel and no kernel of the national line hashes the codecs. What the codecs decode is held by `microcosm-graph`'s codec tests. - The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. Another holds the executor's and key modules' definitions that the kernels' own code names to a listed table, each with a reason: the dtype tokens' map, which the executor casts every node's owned cells through anyway, two artifact declaration records and two capability enums. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: + The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. Another holds the executor's and key modules' definitions that the kernels' own code names to a listed table, each with a reason: the dtype tokens' map, which the executor casts every node's owned cells through anyway, two artifact declaration records, the capabilities record and its five enums, and the kernel context and result records. The kernels name six of them through `microcosm.graph`'s lazy exports, which the walk now follows: the capabilities record, three of its enums, and the context and result records. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: - the seven dense population kernels' shared hash: the engine adapter, the canonical encoder and the weight-update receipt; - the geography identity: `household_lineage`, `source_lineage`, `microcosm.frame.concepts` and the canonical encoder; - property value: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `tenure_constants`, the keyed uniforms and the canonical encoder; @@ -60,4 +62,4 @@ Every UK spine stage now hashes each first-party module its transform computes t - **Held for a ruling.** Nine kernels stay outside the rule, each listed with its reason. Until the ruling, a held kernel can replay a stale output after an edit to a module it computes through but does not hash; a test checks that every held kernel's walk still meets such a module, so a hold cannot outlive its need: - the spine gates, and the full build's preflight and calibrated gates. Each battery shares one key across phases that declare different gates, which it evaluates through `UK_GATE_REGISTRY`. Binding what one phase's evaluators reach would re-key the assembled spine gate, or the preflight and with it the dense solve, on edits that cannot change them. The national gates are held with them at the registry alone: the rule holds everything else they compute through, and the same test checks that the walk past the registry still meets a module their hash leaves out; - the target compilation, selection, measures and problem kernels, and the national targets and problem. microcosm#1497's import record holds their keys and may only shrink. Binding what the rule finds would add to it every unhashed import of each module bound. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Only the size search and refit hash their five modules, and the national line runs neither, so none of its nodes re-keys on an edit to `local_doctrine` or `target_weights` either; its targets and problem kernels compute through the other three and hash them. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 3618ec2985..bde5a459e1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -220,18 +220,29 @@ engine_population, uk_engine_population_declaration, ) +# What else the injected engine computes through when those stages call it: +# the adapter builds each simulation's dataset through frame.materialize's +# engine_tables. test_uk_graph derives the group from a second walk, rooted at +# the PolicyEngineUKEngine the builds inject and followed from the methods the +# engine stages call on it (microcosm#931). It is kept apart from +# _ENGINE_MODULES, which no other stage binds, because the SPI spine's three +# stages and the band donors bind frame.materialize for their own tables. +_ENGINE_COMPUTE_MODULES = (frame_materialize,) +# What every stage that calls the engine binds. +_ENGINE_STAGE_MODULES = (*_ENGINE_MODULES, *_ENGINE_COMPUTE_MODULES) # Imported modules are not traversed by ``source_hash``, so each stage binds # every first-party module holding code its transform computes through, # beyond its own module (microcosm#931). test_uk_graph derives that set: it # follows what the transform names into the module defining it, in any -# workspace package, to a fixed point. It needs no binding for a listed -# refusal-only check, nor for the core modules every node runs through (the -# Frame, the graph executor and the modules that compute node keys), which it -# lists with their reasons. Where it stops at a single string, a column-name -# function or a record, the module is hashed by the stage or a stage upstream -# of it, or the name is listed with its reason; a collection of strings, like -# a number, is a value. +# workspace package, to a fixed point, and for a stage that calls the engine, +# what the methods the stage calls on the injected engine name. It needs no +# binding for a listed refusal-only check, nor for the core modules every node +# runs through (the Frame, the graph executor and the modules that compute +# node keys), which it lists with their reasons. Where it stops at a single +# string, a column-name function or a record, the module is hashed by the +# stage or a stage upstream of it, or the name is listed with its reason; a +# collection of strings, like a number, is a value. # Every stage whose module or bound helpers import the shared keyed-draw # helpers (``microcosm.build.stochastic_assignment``), or import another # module's function that draws through them, binds them as well, so an edit @@ -468,12 +479,12 @@ # The legacy proxies read raw tabs, and the grant split resolves its # year's rules from the FRS release (microcosm#931). "frs_legacy_proxies": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, *_PINNED_TAB_MODULES, stochastic_assignment, ), - "frs_education_grant_split": (*_ENGINE_MODULES, frs_release), - "frs_brma": (*_ENGINE_MODULES, stochastic_assignment), + "frs_education_grant_split": (*_ENGINE_STAGE_MODULES, frs_release), + "frs_brma": (*_ENGINE_STAGE_MODULES, stochastic_assignment), # The donor stages read their donor tabs through frs_spine, record each # fit's weight kind, gate the fits and clip donor support to the donor's # range (microcosm#931); the WAS recipients' predictors map benefit units to @@ -482,7 +493,7 @@ # ETB, landlord, reporter and housing stages below, fit through the QRF # modules (microcosm#931). "was_wealth": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, *_PINNED_TAB_MODULES, *_QRF_MODULES, frs_brma, @@ -498,7 +509,7 @@ # household predictors through the WAS recipient surface; the anchor counts # HMRC's payers through the typed ISA facts (microcosm#1415). "was_lisa": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, was_wealth, hmrc_isa_statistics, *_PINNED_TAB_MODULES, @@ -550,7 +561,7 @@ # The bus travel donors also read published car availability as vendored # facts, and the recipients' HBAI gross income (microcosm#931). "nts_bus_travel": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, bus_use_incidence, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, @@ -570,7 +581,7 @@ # assigns car attributes, tops up gambling and reads the DfT boardings of # each recipient's region through the FRS region map (microcosm#931). "lcfs_consumption": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, energy_pricing, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, @@ -589,7 +600,7 @@ stochastic_assignment, ), "etb_vat": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, *_QRF_MODULES, @@ -603,7 +614,7 @@ # The services stage counts its recipients' family roles as the VAT stage # does, from etb_vat's role table (microcosm#931). "etb_services": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, *_PINNED_TAB_MODULES, *_FEED_FACT_MODULES, *_QRF_MODULES, @@ -623,7 +634,7 @@ # units as the UC capital stage does (microcosm#931). "uc_reporter_redraw": ( uc_relationships, - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, *_QRF_MODULES, qrf_gates, uc_capital_coherence, @@ -661,7 +672,7 @@ # it types its engine as the adapter's ProjectingRulesEngine, a protocol # that extends microcosm.frame.rules' RulesEngine (microcosm#931). "uc_take_up": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, frame_rules, *_FEED_FACT_MODULES, legacy_closure_take_up, @@ -688,7 +699,7 @@ # protocol as the UC redraw (microcosm#931). "pension_credit_take_up": ( uc_capital_coherence, - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, frame_rules, frs_release, take_up_contract, @@ -698,7 +709,7 @@ # reads the vendored Child Benefit statistics (microcosm#931). "child_benefit_take_up": ( uc_capital_coherence, - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, *_FEED_FACT_MODULES, stochastic_assignment, ), @@ -708,7 +719,7 @@ # placement helper (microcosm#1418); it reads the take-up contract and the # year's rules (microcosm#931). "carers_allowance_take_up": ( - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, receipt_counts, frs_release, take_up_contract, @@ -716,7 +727,7 @@ ), # The stored attendance is one engine materialization (microcosm#1095), # whose year's rules come from the FRS release (microcosm#931). - "private_school_attendance": (*_ENGINE_MODULES, frs_release), + "private_school_attendance": (*_ENGINE_STAGE_MODULES, frs_release), # The housing shell keeps council tax bands within the nation, whose # regions are microcosm.calibrate's UK region tier, and gates its fits; # the deduction attributes map benefit units to households and @@ -772,7 +783,7 @@ trace, *_FEED_FACT_MODULES, stochastic_assignment, - *_ENGINE_MODULES, + *_ENGINE_STAGE_MODULES, ), # The copies' childcare re-draw draws through frs_take_up's childcare # helper, reads each household's clone path, and measures its flags for diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 0ceb3d04d5..dba6eb2363 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -779,25 +779,53 @@ def _takes_an_engine(module) -> bool: return False +@functools.cache +def _engine_use() -> tuple[frozenset[str], frozenset[str]]: + """The stages whose module calls the engine, and those whose transform + takes one.""" + from microcosm.build.uk_runtime import graph_kernels + + modules = { + stage: graph_kernels._stage_module(stage) + for stage in graph_kernels._STAGE_MODULES + } + return ( + frozenset( + stage for stage, module in modules.items() if _calls_the_engine(module) + ), + frozenset( + stage for stage, module in modules.items() if _takes_an_engine(module) + ), + ) + + +def _engine_stages() -> frozenset[str]: + """The stages that compute through the engine: those that call it or take + one, less those that take it for its country only.""" + + calling, taking = _engine_use() + return calling | (taking - _ENGINE_TAKEN_FOR_ITS_COUNTRY_ONLY) + + def test_every_stage_that_calls_the_engine_binds_the_engine_modules() -> None: """How an engine call runs, and clone by clone once a frame carries more than one clone, is the adapter's, the shared engine-block machinery's and the UK declaration's (microcosm#931). ``source_hash`` does not follow imports, so every stage whose module calls the engine, or whose transform takes one, binds all three, and no other stage binds any of them in the - stage table. The one exception takes the engine only to check its country - and computes nothing through it. Every stage also hashes the adapter - beside national_frame, which stamps its frames with the adapter's ISA - income contract and savings income projection (the next test).""" + stage table. Each such stage also binds what else the injected engine + computes through for it (``_ENGINE_COMPUTE_MODULES``, which the next test + derives); another stage binds a module of that group only where its own + transform computes through it. The one exception takes the engine only to + check its country and computes nothing through it. Every stage also + hashes the adapter beside national_frame, which stamps its frames with + the adapter's ISA income contract and savings income projection + (test_engine_module_changes_invalidate_every_stage_that_calls_the_engine).""" from microcosm.build.uk_runtime import graph_kernels engine_modules = set(graph_kernels._ENGINE_MODULES) - modules = { - stage: graph_kernels._stage_module(stage) - for stage in graph_kernels._STAGE_MODULES - } - calling = {stage for stage, module in modules.items() if _calls_the_engine(module)} - taking = {stage for stage, module in modules.items() if _takes_an_engine(module)} + compute_modules = set(graph_kernels._ENGINE_COMPUTE_MODULES) + calling, taking = _engine_use() assert { "frs_legacy_proxies", "frs_brma", @@ -808,13 +836,148 @@ def test_every_stage_that_calls_the_engine_binds_the_engine_modules() -> None: } <= calling # The exception is still one: it takes the engine and never calls it. assert _ENGINE_TAKEN_FOR_ITS_COUNTRY_ONLY <= taking - calling - computing = calling | (taking - _ENGINE_TAKEN_FOR_ITS_COUNTRY_ONLY) + computing = _engine_stages() + transform_walks = _transform_walks() for stage in graph_kernels._STAGE_MODULES: bound = set(graph_kernels._STAGE_HELPER_MODULES.get(stage, ())) if stage in computing: - assert engine_modules <= bound, stage + assert engine_modules | compute_modules <= bound, stage else: assert not engine_modules & bound, stage + own = { + module + for module in compute_modules + if module.__name__ in transform_walks[stage].reach + } + assert bound & compute_modules == own, stage + # The SPI spine's three stages and the band donors bind frame.materialize + # for their own tables. + assert { + stage + for stage in set(graph_kernels._STAGE_MODULES) - computing + if compute_modules & set(graph_kernels._STAGE_HELPER_MODULES.get(stage, ())) + } == { + "frs_hmrc_spine_leaves", + "spi_support_channel", + "hmrc_spi_income_spine", + "spi_income_band_donors", + } + + +def test_the_engine_stages_bind_what_the_injected_engine_computes_through() -> None: + """Every stage that calls the engine computes through the + PolicyEngineUKEngine the builds inject, whose code the walk from the + stage's transform does not follow (microcosm#931 ruling 5). A second walk + is rooted at that class (``_ENGINE_CLASS``): it follows the methods the + stage's code names on it, by attribute or through ``getattr``, with the + constructor and every method they call on ``self``, at the first-party + scope with the stage rule's tables (``_engine_walk``). ``_stage_walks`` + adds it to the walk from the transform, so the rule's tests hold what it + reaches like any other module the stage computes through. Every engine + stage names ``materialize``, the RulesEngine protocol's call. Together the + walks reach exactly the three engine modules and + ``_ENGINE_COMPUTE_MODULES``: a module an engine method newly computes + through fails here until it joins the group, which every engine stage + binds (the test above), and so does one that leaves the group.""" + from microcosm.build.uk_runtime import graph_kernels + + roots = _engine_root_walks() + assert set(roots) == _engine_stages() + engine_class = _engine_class() + transform_walks = _transform_walks() + for stage in roots: + methods = _engine_methods(engine_class, transform_walks[stage]) + assert "materialize" in methods, stage + engine_modules = {module.__name__ for module in graph_kernels._ENGINE_MODULES} + compute_modules = { + module.__name__ for module in graph_kernels._ENGINE_COMPUTE_MODULES + } + assert not engine_modules & compute_modules + reached = set().union(*(set(walk.reach) for walk in roots.values())) + assert reached == engine_modules | compute_modules, sorted( + reached ^ (engine_modules | compute_modules) + ) + + +def test_the_engine_root_is_the_engine_the_builds_inject() -> None: + """The second root is the class of the engine both builders construct + and hand the engine stages: graph_kernels' hermetic H2 builder and the + spine build's rules engine (microcosm#931 ruling 5).""" + import ast + + from microcosm.build.uk_runtime import graph_kernels, spine_build + + for module, function in ( + (graph_kernels.__name__, "_fixture_implementations"), + (spine_build.__name__, "_rules_engine"), + ): + node = _module_definitions(module, _HELPER_SCOPE).definitions[function] + built = { + _resolve(module, sub.func.id, _HELPER_SCOPE) + for sub in ast.walk(node) + if isinstance(sub, ast.Call) and isinstance(sub.func, ast.Name) + } + assert _ENGINE_CLASS in built, (module, function) + + +def test_the_engine_root_follows_the_methods_a_stage_calls( + tmp_path, monkeypatch +) -> None: + """Control for the second root: the methods a stage names on an engine, + by attribute on any receiver or through ``getattr``, are followed, with + the constructor and the methods they call on ``self``; a method no stage + names is not, nor what only it computes through.""" + import importlib + import uuid + + scratch = f"engine_root_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / scratch + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + sources = { + "tables": "def tables(frame):\n return frame\n", + "tree": "def tree(year):\n return year\n", + "vocabulary": "def vocabulary():\n return {'a': 1}\n", + "count": "def count(x):\n return int(x)\n", + "engine": ( + "from .count import count\n" + "from .tables import tables\n" + "from .tree import tree\n" + "from .vocabulary import vocabulary\n\n" + "class Engine:\n" + " def __init__(self, clones=1):\n" + " self._clones = count(clones)\n\n" + " def materialize(self, frame, names, period):\n" + " return self._dataset(frame)\n\n" + " def _dataset(self, frame):\n" + " return tables(frame)\n\n" + " def parameters(self, period):\n" + " return tree(period)\n\n" + " def concept_mapping(self):\n" + " return vocabulary()\n" + ), + "stage": ( + "class Transform:\n" + " def __call__(self, frame, model):\n" + " reader = getattr(model, 'parameters', None)\n" + " return model.materialize(frame, ['x'], 2024), reader\n" + ), + } + for module, source in sources.items(): + (package / f"{module}.py").write_text(source) + prefix = f"microcosm.{scratch}." + engine_class = importlib.import_module(f"{prefix}engine").Engine + walk = _helper_walk((f"{prefix}stage", "Transform"), scope=prefix) + methods = _engine_methods(engine_class, walk) + assert methods == {"materialize", "parameters"} + reach = _engine_walk(engine_class, methods, scope=prefix).reach + assert {module.removeprefix(prefix): names for module, names in reach.items()} == { + "engine": {"Engine"}, + "count": {"count"}, + "tables": {"tables"}, + "tree": {"tree"}, + } def test_the_engine_call_scan_sees_any_receiver(tmp_path) -> None: @@ -839,9 +1002,11 @@ def test_the_engine_call_scan_sees_any_receiver(tmp_path) -> None: [ # national_frame stamps every frame through the adapter's ISA income # contract and savings income projection, so every stage hashes it. - ("policyengine_uk", True), - ("engine_population", False), - ("policyengine_uk_population", False), + ("microcosm.frame.adapters.policyengine_uk", True), + ("microcosm.frame.adapters.engine_population", False), + ("microcosm.frame.adapters.policyengine_uk_population", False), + # The adapter builds each simulation's dataset through it. + ("microcosm.frame.materialize", False), ], ) def test_engine_module_changes_invalidate_every_stage_that_calls_the_engine( @@ -851,7 +1016,7 @@ def test_engine_module_changes_invalidate_every_stage_that_calls_the_engine( from microcosm.build.uk_runtime import graph_kernels - module = importlib.import_module(f"microcosm.frame.adapters.{module_name}") + module = importlib.import_module(module_name) graph = uk_spine_graph() registry = uk_registry(graph=graph) stages = sorted( @@ -1007,6 +1172,42 @@ def changed_module_bytes(path): "release's tier from its gate ancestors; node_key binds the declared " "value with the rest of the kernel's capabilities" ), + # The kernels name the entries below through microcosm.graph's lazy + # exports, which the walk follows since microcosm#931 ruling 8. + ("microcosm.graph.kernel", "Capabilities"): ( + "the frozen record of a kernel's declared contract, which a kernel " + "names to declare its capabilities; it only checks the types and " + "consistency of its fields, and node_key binds every field it holds " + "(keys._capabilities_projection)" + ), + ("microcosm.graph.kernel", "Determinism"): ( + "the enum a kernel's capabilities declare its determinism in, whether " + "its output is a function of its inputs and seed; node_key binds the " + "declared value with the rest of the kernel's capabilities" + ), + ("microcosm.graph.kernel", "Numeric"): ( + "the enum a kernel's capabilities declare its numbers' " + "reproducibility in, from which the executor resolves the numeric " + "scope of each column a reader is given; node_key binds the declared " + "value with the rest of the kernel's capabilities" + ), + ("microcosm.graph.kernel", "SeedSource"): ( + "the enum a kernel's capabilities declare its seed source in, where a " + "seeded kernel takes its randomness from; node_key binds the declared " + "value with the rest of the kernel's capabilities" + ), + ("microcosm.graph.kernel", "KernelContext"): ( + "the frozen record of what the executor hands a kernel to read, which " + "a kernel names to annotate it; it only checks and freezes the fields " + "the executor fills, and every node's inputs pass through it, where " + "H2 catches a changed value (graph.kernel's core reason)" + ), + ("microcosm.graph.kernel", "KernelResult"): ( + "the frozen record a kernel returns its outputs in, which holds what " + "the kernel puts in it and computes nothing; the executor validates " + "every field and applies it through graph.population's patch, where " + "H2 catches a changed value as for the executor" + ), } #: Checks that read values only to decide whether to refuse, each with why it #: cannot change what its caller computes. The rule does not follow them, and @@ -1093,11 +1294,22 @@ def changed_module_bytes(path): "resolves the hermetic H2 fixture's transform for the node's stage, " "which _fixture_implementations builds and which is the stage's own " "transform: the stage rule holds what each transform computes " - "through, from where graph_kernels builds it. The one exception is " - "the PolicyEngineUKEngine it builds and injects into the engine " - "stages: they bind the three engine modules, but no rule follows the " - "adapter's own first-party imports, which is held for a ruling " - "(microcosm#931)" + "through, from where graph_kernels builds it, and what the " + "PolicyEngineUKEngine it builds and injects into the engine stages " + "computes through for them, from the methods they call on it (its " + "second root, _ENGINE_CLASS; microcosm#931 ruling 5)" + ), + ("microcosm.graph.codecs", "load_source"): ( + "decodes a declared source into a frame. The create kernel calls it " + "only where no root transform is bound, the hermetic registry's " + "data-only path, which loads the captured root tables of the H2 " + "fixture. The spine build installs the root's transform, and H2 " + "parity binds it too (fixture_stage_plan_inputs), so neither takes " + "that path (checked). Hashing microcosm.graph.codecs, and the content " + "store it names, would re-key the spine root, and with it every spine " + "node and every node downstream of the spine, on edits to code no " + "build runs; microcosm-graph's codec tests hold what it decodes " + "(microcosm#931 ruling 8)" ), ("microcosm.build.uk_runtime.spi_spine", "uk_spi_support_mass_change_reason"): ( "_minimal_frame rebuilds the SPI support allocation's mass record " @@ -1175,6 +1387,22 @@ def changed_module_bytes(path): ) }, } +#: Packages that export names lazily, through a module ``__getattr__``, each +#: with the table, of name to the submodule defining it, that ``__getattr__`` +#: reads. The walk resolves such a name to the module that defines it and +#: follows it like any other (microcosm#931 ruling 8). microcosm.graph imports +#: its runtime modules, which need numpy, pandas and microcosm-frame, only on +#: first access. test_every_lazy_export_resolves_to_the_object_python_returns +#: checks the table. +_LAZY_EXPORT_TABLES: dict[str, str] = {"microcosm.graph": "_RUNTIME_EXPORTS"} +#: First-party modules that define a module ``__getattr__`` but export +#: nothing through it, each with why. +_NO_LAZY_EXPORTS: dict[str, str] = { + "microcosm.build.us_runtime.sources": ( + "its __getattr__ raises for every name: it only words the error for " + "the names it removed" + ), +} #: Method and function names that change their receiver or an argument in #: place. _IN_PLACE_CALLS = frozenset( @@ -1288,6 +1516,8 @@ def _module_definitions(name: str, scope: str) -> _ModuleDefinitions | None: for node in tree.body: if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): definitions[node.name] = node + elif isinstance(node, ast.TypeAlias): + definitions[node.name.id] = node elif isinstance(node, ast.Assign | ast.AnnAssign): targets = node.targets if isinstance(node, ast.Assign) else [node.target] for target in targets: @@ -1297,9 +1527,26 @@ def _module_definitions(name: str, scope: str) -> _ModuleDefinitions | None: return _ModuleDefinitions(imports, modules, tuple(stars), definitions) +@functools.cache +def _lazy_exports(module: str) -> Mapping[str, str]: + """Each name ``module`` exports lazily, to the module defining it, read + from the table its ``__getattr__`` reads (``_LAZY_EXPORT_TABLES``).""" + + import importlib + + table = _LAZY_EXPORT_TABLES.get(module) + if table is None: + return MappingProxyType({}) + exports = getattr(importlib.import_module(module), table) + return MappingProxyType( + {name: f"{module}.{submodule}" for name, submodule in exports.items()} + ) + + def _resolve(module: str, name: str, scope: str) -> tuple[str, str] | None: """The top-level definition ``name`` refers to in ``module``, through - re-exports; None for a module or anything outside ``scope``.""" + re-exports and lazy exports (``_LAZY_EXPORT_TABLES``); None for a module + or anything outside ``scope``.""" pending, seen = [(module, name)], set() while pending: @@ -1315,8 +1562,13 @@ def _resolve(module: str, name: str, scope: str) -> tuple[str, str] | None: return item if name in facts.imports: pending.append(facts.imports[name]) - else: - pending.extend((star, name) for star in facts.stars) + continue + # As in Python, the module's __getattr__ supplies a name only where + # none of its own attributes, its star imports' included, is one. + lazy = _lazy_exports(module).get(name) + if lazy is not None: + pending.append((lazy, name)) + pending.extend((star, name) for star in facts.stars) return None @@ -1478,7 +1730,9 @@ def _classify_helper(module: str, name: str, scope: str) -> str: import ast node = _module_definitions(module, scope).definitions[name] - if isinstance(node, ast.Assign | ast.AnnAssign): + # A type alias statement (``type X = ...``) reads as an assignment of its + # value. + if isinstance(node, ast.Assign | ast.AnnAssign | ast.TypeAlias): if node.value is None or ( ( isinstance(node.value, ast.Name | ast.Attribute) @@ -1626,6 +1880,104 @@ def _helper_reach( return _helper_walk(root, scope).reach +#: The class of the engine the builds inject into the stages that compute +#: through the engine, the second root of their walk (microcosm#931 ruling 5): +#: graph_kernels' hermetic H2 builder and the spine build both construct it +#: (test_the_engine_root_is_the_engine_the_builds_inject). +_ENGINE_CLASS = ("microcosm.frame.adapters.policyengine_uk", "PolicyEngineUKEngine") + + +def _engine_class() -> type: + import importlib + + module, name = _ENGINE_CLASS + return getattr(importlib.import_module(module), name) + + +def _class_attributes(cls: type) -> dict[str, list[tuple[type, object]]]: + """Each attribute ``cls`` defines, with the class and statement that + define it, as ``_kernel_body`` chooses them by method resolution order.""" + + body: dict[str, list[tuple[type, object]]] = {} + for owner, statement, name in _kernel_body(cls): + body.setdefault(name, []).append((owner, statement)) + return body + + +def _engine_methods(engine_class: type, walk: _Walk) -> frozenset[str]: + """The attributes of ``engine_class`` that the code ``walk`` reaches + names: read by attribute, on any receiver, or through ``getattr`` or + ``hasattr`` with a constant name. It matches by name alone, as + ``_calls_the_engine`` matches its calls, so it sees the engine under any + name a stage holds it by, and can only over-count.""" + + import ast + + found: set[object] = set() + for module, names in walk.reach.items(): + definitions = _module_definitions(module, _HELPER_SCOPE).definitions + for name in names: + for sub in ast.walk(definitions[name]): + if isinstance(sub, ast.Attribute): + found.add(sub.attr) + elif ( + isinstance(sub, ast.Call) + and isinstance(sub.func, ast.Name) + and sub.func.id in {"getattr", "hasattr"} + and len(sub.args) > 1 + and isinstance(sub.args[1], ast.Constant) + ): + found.add(sub.args[1].value) + return frozenset(found & set(_class_attributes(engine_class))) + + +def _engine_walk( + engine_class: type, methods: frozenset[str], scope: str = _HELPER_SCOPE +) -> _Walk: + """What an engine of ``engine_class`` computes through for a stage that + calls ``methods`` on it: the stage rule's walk, at ``scope`` with the + same tables, from those methods, the constructor the builds call, and + every method they call on ``self``, to a fixed point.""" + + import ast + + body = _class_attributes(engine_class) + pending, chosen = sorted({"__init__", *methods} & set(body)), set() + while pending: + name = pending.pop() + if name in chosen: + continue + chosen.add(name) + for _, statement in body[name]: + pending.extend( + sub.attr + for sub in ast.walk(statement) + if isinstance(sub, ast.Attribute) + and isinstance(sub.value, ast.Name) + and sub.value.id in {"self", "cls"} + and sub.attr in body + ) + seeds = [ + (found, (owner.__module__, f"{owner.__name__}.{name}")) + for name in sorted(chosen) + for owner, statement in body[name] + for found in _named(owner.__module__, statement, scope) + ] + walk, _, _ = _walk( + {(cls.__module__, cls.__name__) for cls in _kernel_classes(engine_class)}, + seeds, + scope, + ) + return walk + + +def _merged(first: _Walk, second: _Walk) -> _Walk: + reach = dict(first.reach) + for module, names in second.reach.items(): + reach[module] = reach.get(module, frozenset()) | names + return _Walk(reach, first.stops | second.stops, first.core | second.core) + + def _stage_transforms() -> dict[str, tuple[str, str]]: """Each spine stage's transform, as graph_kernels builds it for the hermetic registry and the legacy plan: ``(module, name)``.""" @@ -1668,24 +2020,58 @@ def _stage_transforms() -> dict[str, tuple[str, str]]: return transforms +@functools.cache +def _transform_walks() -> MappingProxyType: + """The walk from each spine stage's transform (``_helper_walk``).""" + + from microcosm.build.uk_runtime import graph_kernels + + walks = {} + for stage, transform in sorted(_stage_transforms().items()): + assert transform[0] == graph_kernels._stage_module(stage).__name__, stage + walks[stage] = _helper_walk(transform) + return MappingProxyType(walks) + + +@functools.cache +def _engine_root_walks() -> MappingProxyType: + """Each engine stage's second walk (``_engine_walk``): from the methods + of the injected engine its transform's walk names.""" + + engine_class = _engine_class() + walks = _transform_walks() + return MappingProxyType( + { + stage: _engine_walk( + engine_class, _engine_methods(engine_class, walks[stage]) + ) + for stage in sorted(_engine_stages()) + } + ) + + @functools.cache def _stage_walks() -> dict[str, tuple[str, frozenset[str], tuple]]: - """Each spine stage's node id, the modules its node hashes, and the - walk from its transform (``_helper_walk``).""" + """Each spine stage's node id, the modules its node hashes, and what it + computes through: the walk from its transform and, for a stage that + calls the engine, the walk from the engine methods it calls + (``_engine_root_walks``).""" from microcosm.build.uk_runtime import graph_kernels graph = uk_spine_graph() registry = uk_registry(graph=graph) + roots = _engine_root_walks() walks = {} - for stage, transform in sorted(_stage_transforms().items()): - assert transform[0] == graph_kernels._stage_module(stage).__name__, stage + for stage, walk in _transform_walks().items(): # The root stage's node is the CREATE kernel. node = graph.node("create_uk_frs" if stage == "frs_spine" else stage) hashed = frozenset( hashed_module_names(registry.get(node.kernel), graph_kernels) ) - walks[stage] = (node.id, hashed, _helper_walk(transform)) + if stage in roots: + walk = _merged(walk, roots[stage]) + walks[stage] = (node.id, hashed, walk) return walks @@ -1705,10 +2091,13 @@ def test_every_stage_hashes_each_helper_its_transform_computes_through() -> None needs its module bound. The rule starts at the transform, not at the kernel that runs it: the kernel's code, in graph_kernels, which every stage hashes, builds every stage's transform for the hermetic registry, - so following it would bind every helper into every stage. + so following it would bind every helper into every stage. For a stage + that calls the engine it also starts at the methods the stage calls on + the engine the builds inject (``_engine_root_walks``). - What the rule cannot see: a call through an object whose class the code - never names, a name looked up by string, and data files read by path. + What the rule cannot see: a call through any other object whose class + the code never names, a name looked up by a computed string, and data + files read by path. """ from microcosm.build.uk_runtime import graph_kernels @@ -1849,11 +2238,11 @@ def test_the_core_modules_are_on_every_nodes_path() -> None: kernel = registry.get(graph.node(node_id).kernel) # regional_home_values draws its keyed uniforms through # microcosm.graph.randomness, whose contract asks a kernel drawing - # through it to declare SeedSource.KEYED. It declares NONE, as every - # spine stage kernel does through the one stage kernel class: its - # stream is a stage parameter, which its contract hash covers, and it - # reads no generator the executor seeds. Whether it should declare - # KEYED is held for a ruling (microcosm#931 review). + # through it to declare SeedSource.KEYED. Every spine stage kernel, + # its own included, declares NONE through the one stage kernel class, + # as María ruled on 9 Nov (microcosm#931 ruling 7): its stream is a + # stage parameter, which its contract hash covers, and it reads no + # generator the executor seeds. assert kernel.capabilities.seed_source is SeedSource.NONE, stage @@ -1907,21 +2296,35 @@ def test_the_refusal_check_sees_each_way_a_function_writes() -> None: def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( tmp_path, monkeypatch ) -> None: - """Control: a helper called by name, through a re-export, by attribute on - its module, through an alias or a computed constant, or held only in an - annotation (a record's field type included) is reached, as are a numeric - constant, a collection of strings and a function that writes in place or - only refuses but is not listed; a listed check, a single string, a column - name, a record of plain fields, a listed core module, what only a core - module names, and a helper the root never names are not. The walk returns - the names it stopped at and the core modules it met, and follows a core - module only when asked to.""" + """Control: a helper called by name, through a re-export, a lazy export, + by attribute on its module, through an alias or a computed constant, or + held only in an annotation (a record's field type included) is reached, + as are a numeric constant, a collection of strings and a function that + writes in place or only refuses but is not listed; a listed check, a + single string, a column name, a record of plain fields, a listed core + module, what only a core module names, and a helper the root never names + are not. The walk returns the names it stopped at and the core modules it + met, and follows a core module only when asked to.""" import uuid scratch = f"helper_rule_{uuid.uuid4().hex[:8]}" package = tmp_path / "microcosm" / scratch package.mkdir(parents=True) (package / "__init__.py").write_text("") + # A package exporting its runtime module's names lazily, as + # microcosm.graph does. + (package / "lazily").mkdir() + (package / "lazily" / "__init__.py").write_text( + "from importlib import import_module\n\n" + "_EXPORTS = {'deferred': 'runtime', 'idle': 'runtime'}\n\n" + "def __getattr__(name):\n" + " if name not in _EXPORTS:\n" + " raise AttributeError(name)\n" + " return getattr(import_module(f'{__name__}.{_EXPORTS[name]}'), name)\n" + ) + (package / "lazily" / "runtime.py").write_text( + "def deferred(x):\n return x * 6\n\ndef idle(x):\n return x * 7\n" + ) monkeypatch.syspath_prepend(str(tmp_path)) sources = { "direct": "def draw(x):\n return x + 1\n", @@ -1983,6 +2386,7 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( "from .guard import guard\n" "from .harmless import COLUMN, Record, check, column\n" "from .inplace import fold\n" + "from .lazily import deferred\n" "from .numeric import FLOOR\n" "from .reexport import total\n" "from .unused import never\n\n" @@ -1993,7 +2397,7 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( " check(frame)\n" " guard(frame)\n" " fold(frame, [0], [1])\n" - " record = Record(value=draw(1) + total([1]))\n" + " record = Record(value=draw(1) + total([1]) + deferred(1))\n" " rate = self.contract.rate(column('household'))\n" " rate += self.inputs.rate.value()\n" " owners = [OWNER[code] for code in CODES]\n" @@ -2012,11 +2416,13 @@ def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( _REFUSAL_ONLY_HELPERS, (f"{prefix}harmless", "check"), "control" ) monkeypatch.setitem(_CORE_MODULES, f"{prefix}core", "control") + monkeypatch.setitem(_LAZY_EXPORT_TABLES, f"{prefix}lazily", "_EXPORTS") reach, stops, core = _helper_walk((f"{prefix}stage", "Transform"), scope=prefix) assert {module.removeprefix(prefix): names for module, names in reach.items()} == { "stage": {"Transform"}, "direct": {"draw"}, "origin": {"total"}, + "lazily.runtime": {"deferred"}, "attribute": {"share"}, "alias": {"scale", "_scale"}, "computed": {"TABLE", "_build"}, @@ -2279,13 +2685,12 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: refusal-only and core-module tables as for the stages, and at ``_KERNEL_STOPS``. Where a class has several kernels (the spine's stage kernels, one per stage), each one's key is held to it. The held kernels - (``_HELD_KERNELS``) are not. - - What the rule cannot see, beside what the stage rule cannot: a name a - package exports lazily, through its module ``__getattr__``. The one a - kernel calls is microcosm.graph's ``load_source``, through which the - create kernel loads the H2 fixture's tables when no root transform is - bound; the production spine always binds one.""" + (``_HELD_KERNELS``) are not. A name microcosm.graph exports lazily, + through its module ``__getattr__``, is followed to the runtime module + that defines it like any other (``_LAZY_EXPORT_TABLES``): the kernels' + capability and context records are listed core names, and the create + kernel's ``load_source``, which it calls only where no root transform is + bound, is a listed stop.""" from microcosm.build.uk_runtime import graph_population, rowwise_geography kernels = _uk_graph_kernels() @@ -2400,6 +2805,41 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: hashed = _hashed_modules(kernels[f"uk.stage.{stage}@1"]) assert set(reach.reach) | {module for module, _ in reach.stops} <= hashed + # The source codecs: only the create kernel's run loads a source through + # them, and only inside its branch for an unbound root transform. The + # spine build installs the root's transform, and fixture_stage_plan_inputs, + # which H2 parity builds its registry from, adds it. + loading = ("microcosm.graph.codecs", "load_source") + assert namers[loading] == { + "uk.create@1": frozenset({(graph_kernels.__name__, "UKCreateKernel.run")}) + } + (run,) = [ + item + for item in definitions["UKCreateKernel"].body + if isinstance(item, ast.FunctionDef) and item.name == "run" + ] + unbound = set() + for branch in ast.walk(run): + if ( + isinstance(branch, ast.If) + and ast.unparse(branch.test) == "self.transform is None" + ): + unbound |= {id(sub) for item in branch.body for sub in ast.walk(item)} + references = [ + sub + for sub in ast.walk(run) + if isinstance(sub, ast.Name | ast.Attribute) + and loading in _named(graph_kernels.__name__, sub, _HELPER_SCOPE) + ] + assert references and all(id(sub) in unbound for sub in references) + assert "frs_spine" in {install.stage for install in _spine_build_installs()} + plan = definitions["fixture_stage_plan_inputs"] + assert any( + isinstance(sub, ast.Assign) + and ast.unparse(sub.targets[0]) == "implementations['frs_spine']" + for sub in ast.walk(plan) + ) + # The gate registry: the bound spine derives its gate digests through it, # in _manifest_digests, which returns the battery run's digests alone; # the national battery evaluates through it and is held there. @@ -2473,6 +2913,89 @@ def test_the_core_definitions_the_kernels_name_are_listed() -> None: assert "dtype_for_token" in patching.reach["microcosm.graph.population"] +def test_every_lazy_export_resolves_to_the_object_python_returns() -> None: + """The walk follows a name a package exports lazily, through its module + ``__getattr__``, to the module that defines it, as Python imports it + (microcosm#931 ruling 8). Every name microcosm.graph exports that way + resolves to a definition holding the very object the package returns, + the table the walk reads is the one the package's ``__getattr__`` reads, + and a name the package imports eagerly resolves as before. Every + first-party module that defines a module ``__getattr__`` is listed, as a + lazy exporter or as one that exports nothing through it, so a new one + cannot hide names from the walk.""" + import ast + import importlib + + import microcosm + from microcosm import graph + + for module, table in _LAZY_EXPORT_TABLES.items(): + getter = _module_definitions(module, _HELPER_SCOPE).definitions["__getattr__"] + assert table in { + sub.id for sub in ast.walk(getter) if isinstance(sub, ast.Name) + }, module + exports = getattr(graph, _LAZY_EXPORT_TABLES[graph.__name__]) + assert {"load_source", "KernelContext", "keyed_uniform"} <= set(exports) + for name in exports: + found = _resolve(graph.__name__, name, _HELPER_SCOPE) + assert found is not None, name + module, defined = found + assert getattr(importlib.import_module(module), defined) is getattr( + graph, name + ), name + assert _resolve(graph.__name__, "compile_graph", _HELPER_SCOPE) == ( + "microcosm.graph.decl", + "compile_graph", + ) + getters = set() + for root in map(Path, microcosm.__path__): + for path in sorted(root.rglob("*.py")): + text = path.read_text(encoding="utf-8") + if "def __getattr__" not in text or not any( + isinstance(node, ast.FunctionDef) and node.name == "__getattr__" + for node in ast.parse(text).body + ): + continue + parts = path.relative_to(root.parent).with_suffix("").parts + getters.add(".".join(parts[:-1] if parts[-1] == "__init__" else parts)) + listed = set(_LAZY_EXPORT_TABLES) | set(_NO_LAZY_EXPORTS) + assert getters == listed, sorted(getters ^ listed) + # The modules that export nothing that way only raise from __getattr__. + for module in _NO_LAZY_EXPORTS: + getter = _module_definitions(module, _HELPER_SCOPE).definitions["__getattr__"] + assert not any(isinstance(sub, ast.Return) for sub in ast.walk(getter)) + + +def test_no_spine_or_national_line_kernel_hashes_the_source_codecs() -> None: + """The create kernel's ``load_source`` stop keeps the spine's keys off + microcosm.graph.codecs (microcosm#931 ruling 8): no spine kernel hashes + it, so an edit to it re-keys no spine node, nor any node downstream of + the spine through it, and no kernel of the national line, whose root is + the bound spine checkpoint, hashes it either. The atomic geography + kernels, which read their sources' bytes through it, do.""" + from microcosm.build.uk_runtime import ( + graph_build, + graph_calibration, + graph_national, + ) + from microcosm.graph import codecs + + spine = uk_registry(graph=uk_spine_graph()) + for ref in spine.refs(): + assert codecs.__name__ not in _hashed_modules(spine.get(ref)), ref + kernels = _uk_graph_kernels() + for ref in ( + graph_build.UKBoundSpineKernel.ref, + graph_national.UKNationalTargetKernel.ref, + graph_national.UKNationalProblemKernel.ref, + graph_calibration.UKDenseSolveKernel.ref, + graph_national.UKNationalGateKernel.ref, + graph_national.UKNationalReadbackKernel.ref, + ): + assert codecs.__name__ not in _hashed_modules(kernels[ref]), ref + assert codecs.__name__ in _hashed_modules(kernels["geography.support_import@1"]) + + def test_the_frame_totals_the_uk_corrections_target_are_numpys_sums() -> None: """H2 cannot catch an edit to the Frame, which the graph's spine and the legacy stage plan both run through (``_FRAME_REASON``). The UK exact-total From a509ae2b11de53c32d41f22a329725f1ed117781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:09:13 +0000 Subject: [PATCH 037/108] Key each gate battery phase on what its own gates compute through (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María ruled on 9 Nov (ruling 6) that each phase of a gate battery keys on what its own gates compute through, with what only one phase computes through carried on that phase's params, and that the single-phase national gates bind the gate registry's whole reach. This commit does both. It moves keys only, and the full gates now refuse a stale preflight code as they refused a stale terminal code. - The model. test_uk_graph derives what each battery phase computes through (_battery_phase_walk). It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase, read from UK_GATE_REGISTRY's literal mapping. In stage_health it follows only the checks the phase declares, by each entry's `check`. In the spine gates' artifact table (uk_spine_gate_artifacts) it follows only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. The kernel rule also leaves out a method only the implementation hash calls, the full gates' _country_resources. Another test checks each part of the model against the code. - The check. A battery's shared hash holds what every phase computes through and nothing only some phases do. Each phase's own code, carried as `_code`, holds exactly the rest. The kernel rule's test holds each phase's key to cover what it computes through. - The spine gates. The shared hash has 11 modules. It gains gates, stage_evidence, graph_kernels, national_frame, receipt_counts, council_tax_constants and calibrate.geography_constants, and stochastic_assignment moves to the transferred phase. The assembled node carries a new assembled_code (frs_relationships). energy_pricing_code becomes transferred_code, 17 modules. As before, the spine gate kernel does not check them. - The full gates. The shared hash has 26 modules. local_rowwise, weighted_integrity and diagnostics move to terminal_code. high_value_homes and calibrate.geography_constants move into the shared hash, since the preflight's target surface and ledger parity compute through them. The preflight node carries a new preflight_code (microcosm.build.ledger_targets), and terminal_code hashes 37 modules. The kernel refuses a stale or missing code in either phase. - The national gates. Their hash adds the 50 modules battery_bindings.UK_GATE_REGISTRY_MODULES lists, which a test holds to the registry's derived reach. Their held stop goes, and _HELD_KERNEL_STOPS with it. The three batteries leave _HELD_KERNELS, which keeps the six target kernels. - Listed with reasons: rowwise_dataset's clone index column name, which the calibrated and national gates' support-family fold reads, and source_hash, a core name the full gates' code check reaches. - Tests and text. The spine evidence and terminal tests follow the new params. The stale code test covers both full phases and a missing code, and a new test checks each full gate node's code. The control test pins the key-only method. The changelog fragment moves the batteries and the national gates from held to bound. What moves: node keys only. No value, declaration or generated file moves. - 74 of the 99 kernel hashes move. graph_evidence.py changed, and 65 kernels hash it: every spine stage and expand kernel, uk.create, the seven dense population kernels, the spine gates, the preflight and calibrated gates and the certification. graph_terminal.py also moves the export, package, inspection and holdout kernels. graph_national.py moves the national targets, problem, gates and readback. - So every node of the gated spine re-keys (81 of 81), and every node of the country adapter's in-graph build (113 of 113). On a bound spine checkpoint, every node after the checkpoint re-keys, the preflight and the dense solve among them. On the national line 5 of 6 nodes re-key; the bound checkpoint keeps its key. The transport graphs keep theirs. - Later edits: one to a module only the transferred phase computes through re-keys the transferred gate, not the assembled gate or the stages that wait on it. One to a module only the terminal phase computes through re-keys the calibrated gates and what follows, not the preflight or the solve. On a bound spine, edits to stage_evidence, parity_reference, release_identity and release_input_coverage now re-key the preflight, and so re-solve: the preflight's release input coverage gate computes through them, and its key used to miss them. - No fixture or generated file holds a gate node. uk_spine.json, the H2 fixture, source_stages.json and the coverage manifest do not move. Tests: test_uk_graph, test_uk_graph_evidence and test_uk_graph_terminal pass (98 tests). 25 related engine-free files pass (1,276 tests) but for one test that also fails at the parent on this machine (test_uk_cgt_projection asserts the UK engine is not installed). Through the engine slot, the engine terminal and full-graph admission tests pass, the H2 fixture regenerates byte-identically under --check, and test_h2_uk_spine_parity passes. 17 mutations under scratch are caught: a phase newly computing through a module its key misses (a transferred check or the terminal branch reading an unhashed module, an assembled check reading a transferred-only one, the preflight branch reading a terminal-only one), a phase-only module moved into a shared hash, a shared module moved into one phase's code, a terminal-only module put in the preflight's code, the national gates dropping a registry module or the registry's reach, an unmodelled stage-health branch, another evaluator reading a domain artifact, the run calling the key-only method, the run dropping its code check, and a spine node dropping its code. Ruff is clean on the seven edited Python files, and source_stages.json and the coverage manifest are current. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-helper-bindings.changed.md | 23 +- .../build/uk_runtime/battery_bindings.py | 73 +- .../build/uk_runtime/graph_evidence.py | 149 ++- .../build/uk_runtime/graph_national.py | 24 +- .../build/uk_runtime/graph_terminal.py | 255 ++++- .../tests/engine_free/uk/test_uk_graph.py | 988 +++++++++++++++--- .../engine_free/uk/test_uk_graph_evidence.py | 42 +- .../engine_free/uk/test_uk_graph_terminal.py | 93 +- 8 files changed, 1366 insertions(+), 281 deletions(-) diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 245479ca6a..cb4f66054d 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,4 +1,4 @@ -Every UK spine stage now hashes each first-party module its transform computes through, and each stage that calls the engine also each one the injected engine computes through for it. Every other UK graph kernel, except nine held for a ruling, hashes each one its own code computes through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. +Every UK spine stage now hashes each first-party module its transform computes through, and each stage that calls the engine also each one the injected engine computes through for it. Every other UK graph kernel, except six held for a ruling, hashes each one its own code computes through, and each phase of a gate battery each one its own gates compute through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. - **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage, and for a stage that calls the engine also at the engine the builds inject (see The engine), and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports, module attributes and `type` aliases, which read as assignments of their values. So do the names `microcosm.graph` exports lazily through its module `__getattr__`: the walk resolves each to the runtime module that defines it, from the table that `__getattr__` reads, and a test checks that every first-party module with a module `__getattr__` is listed. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: - six listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels); @@ -35,13 +35,13 @@ Every UK spine stage now hashes each first-party module its transform computes t The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input`, `gates` and `spi_support`, whose replacement strata the sample reads. - **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. Through `graph_implementation_dependencies`, which the root node hashes, it declares `national_sampling` at every rung, since it resets the root's indexes at f100 too, and `frame_sampling` only below f100, where it samples. Its behaviour is unchanged. -- **Kernels.** A second test applies the rule to every kernel the UK registries register but the nine held below: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; and the national gates and readback. It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at four listed kernel stops, each with a reason a test checks: +- **Kernels.** A second test applies the rule to every kernel the UK registries register but the six held below: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; the national gates and readback; and, phase by phase, the spine gates and the full build's preflight and calibrated gates (see Gate batteries). It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key, or a method only that hash calls, such as the full gates' `_country_resources`. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at four listed kernel stops, each with a reason a test checks: - the hermetic fixture's transform resolver, since the stage rule holds each stage's transform and, through its second root, the engine the resolver's builder injects into the engine stages (see The engine); - - `_minimal_frame`'s rebuild of the SPI support allocation's mass record, which only `hmrc_spi_income_spine` takes, and whose reach that stage's node hashes; - - `UK_GATE_REGISTRY`, which the bound spine reaches only for its checkpoint's gate digests. The national gates evaluate through it and are held there, in a table of their own (see Held for a ruling); + - `_minimal_frame`'s rebuild of the SPI support allocation's mass record, which only `hmrc_spi_income_spine` takes, and whose reach that stage's node hashes. The spine gates rebuild their frame through `_minimal_frame` too, and no spine gate node names a stage (checked); + - `UK_GATE_REGISTRY`, which the bound spine reaches only for its checkpoint's gate digests. The gate batteries and the national gates evaluate through it, and the rule holds them past it (see Gate batteries); - `microcosm.graph.codecs`' `load_source`, which the walk now sees through `microcosm.graph`'s lazy exports. The create kernel calls it only where no root transform is bound, the hermetic registry's data-only path, which loads the H2 fixture's captured root tables. The spine build installs the root's transform and H2 parity binds it too, so neither takes that path (checked). Binding the codecs, and the content store they name, would re-key the spine root, and with it every spine node and every node downstream of the spine, on edits to code no build runs. As María ruled on 9 Nov, it is a stop instead, and a test checks that no spine kernel and no kernel of the national line hashes the codecs. What the codecs decode is held by `microcosm-graph`'s codec tests. - The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. Another holds the executor's and key modules' definitions that the kernels' own code names to a listed table, each with a reason: the dtype tokens' map, which the executor casts every node's owned cells through anyway, two artifact declaration records, the capabilities record and its five enums, and the kernel context and result records. The kernels name six of them through `microcosm.graph`'s lazy exports, which the walk now follows: the capabilities record, three of its enums, and the context and result records. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: + The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. Another holds the executor's and key modules' definitions that the kernels' own code names to a listed table, each with a reason: the dtype tokens' map, which the executor casts every node's owned cells through anyway, two artifact declaration records, the capabilities record and its five enums, the kernel context and result records, and `source_hash`, through which the full gates' run recomputes its phase's own code only to refuse a stale one. The kernels name six of them through `microcosm.graph`'s lazy exports, which the walk now follows: the capabilities record, three of its enums, and the context and result records. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: - the seven dense population kernels' shared hash: the engine adapter, the canonical encoder and the weight-update receipt; - the geography identity: `household_lineage`, `source_lineage`, `microcosm.frame.concepts` and the canonical encoder; - property value: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `tenure_constants`, the keyed uniforms and the canonical encoder; @@ -55,11 +55,14 @@ Every UK spine stage now hashes each first-party module its transform computes t - the export readback: `artifact_files`, `trace`, `geography_ladder`, the engine adapter, `frame.materialize` and the canonical encoder; the package inventory: `artifact_files`, `trace` and the canonical encoder; - the certification: `artifact_files`, `trace`, `gate_battery`, `gates`, `graph_evidence`, `calibration_run` and the canonical encoder; - the rotated holdout: the 35 modules that its problem reconstruction, folds and rowwise solves compute through. They are the local surface's modules, `holdout`, `local_target_census`, `size_checkpoint`, `solve_progress`, `weighted_integrity`, microcosm.calibrate's solver modules, `gate_battery`, `stage_evidence`, the engine adapter and the canonical encoder; - - the national gates: `gates`, microcosm.calibrate's `artifacts`, `matrix`, `registry`, `target`, `hierarchy`, `monetary_binding` and `solve`, `graph_targets`, `graph_population`, `national_frame`, `etb_services`, `cgt_projection`, `donor_uprating` and the canonical encoder; + - the national gates: `gates`, microcosm.calibrate's `artifacts`, `matrix`, `registry`, `target`, `hierarchy`, `monetary_binding` and `solve`, `graph_targets`, `graph_population`, `national_frame`, `etb_services`, `cgt_projection`, `donor_uprating` and the canonical encoder, and everything the gate registry's bindings compute through (see Gate batteries); - the national readback: `artifact_files`, `trace`, `frame.materialize`, the engine adapter, `geography_ladder`, `graph_population`, `hmrc_uprating` and the canonical encoder. - The LAD23 names' pin stays listed, now for every kernel that reads it. `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside the national gates' and readback's hashes. -- **Held for a ruling.** Nine kernels stay outside the rule, each listed with its reason. Until the ruling, a held kernel can replay a stale output after an edit to a module it computes through but does not hash; a test checks that every held kernel's walk still meets such a module, so a hold cannot outlive its need: - - the spine gates, and the full build's preflight and calibrated gates. Each battery shares one key across phases that declare different gates, which it evaluates through `UK_GATE_REGISTRY`. Binding what one phase's evaluators reach would re-key the assembled spine gate, or the preflight and with it the dense solve, on edits that cannot change them. The national gates are held with them at the registry alone: the rule holds everything else they compute through, and the same test checks that the walk past the registry still meets a module their hash leaves out; + The LAD23 names' pin stays listed, now for every kernel that reads it. The geographic clone index columns' name (`rowwise_dataset.ladder_clone_index_column`) is listed for the calibrated and national gates, whose support-family fold reads it: only the dense expand writes those columns, and it hashes `rowwise_dataset`; the national frame has no ladder, and the fold keys on the column only where the table carries it. `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside the national gates' and readback's hashes. +- **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Another test checks each part of that model against the code. So an edit only the transferred phase computes through re-keys neither the assembled gate nor the spine stages that wait on it, and one only the terminal phase computes through re-keys neither the preflight nor the dense solve keyed on it. + - The spine gates' hash binds the 11 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants` and `calibrate.geography_constants`, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. As before, the spine gate kernel does not check its phase's code. + - The full gates' hash binds the 26 modules both phases compute through. It loses `local_rowwise`, `weighted_integrity` and `diagnostics`, which only the terminal phase computes through. The preflight node carries a new `preflight_code`: the shared ledger's registry parity, behind the ledger compile parity gates only the preflight declares. The calibrated node's `terminal_code` now hashes 37 modules. `high_value_homes` and `calibrate.geography_constants` move into the shared hash, since the preflight computes through them too; the target kernels hash both, so an edit to either already re-solved. The kernel refuses a stale or missing `preflight_code`, as it refuses a stale `terminal_code`. + - The national gates evaluate whichever gates their node's scope names, so their hash binds everything the gate registry's bindings compute through. That is 50 modules, which `battery_bindings.UK_GATE_REGISTRY_MODULES` lists and a test holds exactly to the registry's derived reach. Their stop at the registry, the one held kernel stop, goes. +- **Held for a ruling.** Six kernels stay outside the rule, each listed with its reason. Until the ruling, a held kernel can replay a stale output after an edit to a module it computes through but does not hash; a test checks that every held kernel's walk still meets such a module, so a hold cannot outlive its need: - the target compilation, selection, measures and problem kernels, and the national targets and problem. microcosm#1497's import record holds their keys and may only shrink. Binding what the rule finds would add to it every unhashed import of each module bound. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, so neither regenerates. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 2f68553bf6..25cd1b25a0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -28,12 +28,13 @@ from __future__ import annotations import hashlib +import importlib import json from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, replace from datetime import date, datetime from importlib.resources import files -from types import MappingProxyType +from types import MappingProxyType, ModuleType from typing import Any import numpy as np @@ -133,7 +134,9 @@ __all__ = [ "UK_GATE_REGISTRY", + "UK_GATE_REGISTRY_MODULES", "UKGateBinding", + "uk_gate_registry_modules", ] @@ -2463,3 +2466,71 @@ def _ledger_compile_parity_required_artifacts( evidence=_ledger_compile_parity_evidence, ), } + +#: Every first-party module the registry's bindings compute through: each +#: binding's evaluator and hooks, what they call, to a fixed point, this module +#: among them. A kernel that evaluates whichever gates its node names through +#: the registry keys on them all: the national gates, whose node's gate scope +#: names their gates (microcosm#931 ruling 6). The gate batteries, whose +#: phases declare their gates, key each phase on its own gates' share of +#: these instead. test_uk_graph derives the set from the registry and holds +#: this list to it exactly. +UK_GATE_REGISTRY_MODULES: tuple[str, ...] = ( + "microcosm.build.chronicle_epoch", + "microcosm.build.gate_battery", + "microcosm.build.gates", + "microcosm.build.input_mass", + "microcosm.build.ledger_targets", + "microcosm.build.stochastic_assignment", + "microcosm.build.uk_runtime.battery_bindings", + "microcosm.build.uk_runtime.bus_fare_pricing", + "microcosm.build.uk_runtime.bus_support_pricing", + "microcosm.build.uk_runtime.bus_use_incidence", + "microcosm.build.uk_runtime.cgt_projection", + "microcosm.build.uk_runtime.chronicle_feed", + "microcosm.build.uk_runtime.council_tax_constants", + "microcosm.build.uk_runtime.dfe_spring_term", + "microcosm.build.uk_runtime.diagnostics", + "microcosm.build.uk_runtime.donor_uprating", + "microcosm.build.uk_runtime.energy_pricing", + "microcosm.build.uk_runtime.etb_services", + "microcosm.build.uk_runtime.frs_annual_income", + "microcosm.build.uk_runtime.frs_take_up", + "microcosm.build.uk_runtime.gambling_participation", + "microcosm.build.uk_runtime.geography_ladder", + "microcosm.build.uk_runtime.high_value_home_gates", + "microcosm.build.uk_runtime.high_value_homes", + "microcosm.build.uk_runtime.hmrc_capital_gains", + "microcosm.build.uk_runtime.lcfs_consumption", + "microcosm.build.uk_runtime.ledger_fact_vendoring", + "microcosm.build.uk_runtime.ledger_targets", + "microcosm.build.uk_runtime.local_authority_input", + "microcosm.build.uk_runtime.local_targets", + "microcosm.build.uk_runtime.national_frame", + "microcosm.build.uk_runtime.nts_bus_travel", + "microcosm.build.uk_runtime.parity_reference", + "microcosm.build.uk_runtime.receipt_counts", + "microcosm.build.uk_runtime.release_identity", + "microcosm.build.uk_runtime.release_input_coverage", + "microcosm.build.uk_runtime.rowwise_geography", + "microcosm.build.uk_runtime.source_runtime", + "microcosm.build.uk_runtime.stage_health", + "microcosm.build.uk_runtime.take_up_contract", + "microcosm.build.uk_runtime.terminal_gates", + "microcosm.build.uk_runtime.uc_capital_coherence", + "microcosm.build.uk_runtime.weighted_integrity", + "microcosm.calibrate.geography_constants", + "microcosm.calibrate.hierarchy", + "microcosm.calibrate.registry", + "microcosm.calibrate.solve", + "microcosm.calibrate.target", + "microcosm.frame.adapters.policyengine_uk", + "microcosm.frame.materialize", +) + + +def uk_gate_registry_modules() -> tuple[ModuleType, ...]: + """The modules :data:`UK_GATE_REGISTRY_MODULES` names, imported, for a + kernel's implementation hash.""" + + return tuple(importlib.import_module(name) for name in UK_GATE_REGISTRY_MODULES) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py index be42907950..3830260865 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -4,8 +4,10 @@ import hashlib import json -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import replace +from functools import cache +from types import MappingProxyType from microcosm.graph import ( ArtifactInput, @@ -26,7 +28,7 @@ source_hash, ) -from .. import gate_battery, stochastic_assignment +from .. import gate_battery from ..country_spec import CountrySpec, GatesManifest from ..gate_battery import ( BlockingMode, @@ -41,7 +43,7 @@ decode_stage_evidence, encode_stage_evidence, ) -from . import battery_bindings, energy_pricing, stage_health +from . import battery_bindings, stage_health from .calibration_run import UK_SPINE_GATE_SCOPE, uk_scoped_gate_manifest from .cgt_asset_type import CGT_ASSET_TYPE_DOMAIN from .frs_relationships import CHRONICLE_ONS_HOUSEHOLD_TYPE_VALUE_IDS @@ -174,6 +176,95 @@ def uk_spine_gate_artifacts(engine: object) -> dict[str, object]: } +@cache +def uk_spine_assembled_only_code_hash() -> str: + """The source hash of the code only the assembled spine phase computes + through: the ONS household type domain its enum gate checks + (``frs_relationships``), which no transferred-phase gate reads. + + Hashed once per process, as for the calibrated gates' own code + (``graph_terminal.uk_terminal_only_code_hash``). + """ + from . import frs_relationships + + return source_hash(frs_relationships) + + +@cache +def uk_spine_transferred_only_code_hash() -> str: + """The source hash of the code only the transferred spine phase computes + through, which the assembled gate's key leaves out. + + That is what the stage-health checks only the transferred phase declares + compute through: the energy rake's published level and NEED margins + (``energy_pricing``); the bus travel, bus pricing, bus support pricing, + consumption basis and gambling checks' published facts and pricing + (``nts_bus_travel``, ``bus_use_incidence``, ``bus_fare_pricing``, + ``bus_support_pricing``, ``etb_services``, ``lcfs_consumption``, + ``donor_uprating``, ``gambling_participation``, the shared ledger's + calendar-year weights, and the vendored-fact reader with the chronicle + feed and its schema epochs); the annual income spells check + (``frs_annual_income``); the apprentice draw's take-up contract + (``take_up_contract``); and the latent-attribute checks' binomial band + (``stochastic_assignment``). Its + enum gate also checks the CGT asset type domain (``cgt_asset_type``). The + assembled gate, which every later spine stage waits on, therefore re-keys + on none of them (the re-review of microcosm#1420, E; microcosm#931 ruling + 6). Hashed once per process, as the assembled phase's own code is. + """ + from .. import chronicle_epoch, stochastic_assignment + from .. import ledger_targets as shared_ledger_targets + from . import ( + bus_fare_pricing, + bus_support_pricing, + bus_use_incidence, + cgt_asset_type, + chronicle_feed, + donor_uprating, + energy_pricing, + etb_services, + frs_annual_income, + gambling_participation, + lcfs_consumption, + ledger_fact_vendoring, + nts_bus_travel, + take_up_contract, + ) + + return source_hash( + energy_pricing, + lcfs_consumption, + bus_fare_pricing, + bus_support_pricing, + bus_use_incidence, + nts_bus_travel, + etb_services, + gambling_participation, + donor_uprating, + frs_annual_income, + take_up_contract, + cgt_asset_type, + stochastic_assignment, + chronicle_feed, + chronicle_epoch, + ledger_fact_vendoring, + shared_ledger_targets, + ) + + +#: Each spine gate phase's own code: the hash of what only that phase's gates +#: compute through, which its node carries as its ``_code`` parameter, +#: so an edit there re-keys that phase alone. The code both phases compute +#: through is the gate kernel's own hash (microcosm#931 ruling 6); +#: test_uk_graph derives each phase's share from the gates it declares. +UK_SPINE_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( + { + "assembled": uk_spine_assembled_only_code_hash, + "transferred": uk_spine_transferred_only_code_hash, + } +) + + def load_spine_stage_artifacts( manifest, store, *, stage_names: Sequence[str] ) -> dict[str, dict[str, object]]: @@ -260,6 +351,7 @@ def add_uk_spine_gate_nodes( for phase, population, inputs, phase_stages in checkpoints: if phase == "transferred" and end == len(stages): continue + code = UK_SPINE_GATE_PHASE_CODE.get(phase) nodes.append( Node( id=f"spine.gates.{phase}", @@ -294,11 +386,7 @@ def add_uk_spine_gate_nodes( "phase": phase, "time_period": "2024", "stage_names": phase_stages, - **( - {"energy_pricing_code": source_hash(energy_pricing)} - if phase == "transferred" - else {} - ), + **({f"{phase}_code": code()} if code is not None else {}), "gate_manifest": json.dumps( gate_battery._gates_manifest_payload(gates), sort_keys=True ), @@ -350,9 +438,11 @@ class UKSpineGateKernel(KernelBase): the persisted verdict binds the policy it was judged under: ``synthetic_smoke`` mirrors ``GateBatteryRun(synthetic_smoke=...)`` — on a data-only fixture the population-fact gates are recorded but never block. - The transferred phase also carries ``energy_pricing_code``, the hash of - the module that sets the energy level its rake gate recomputes, so only - that phase re-keys when the level's code changes. + Each phase also carries ``_code``, the hash of the code only its + own gates compute through (:data:`UK_SPINE_GATE_PHASE_CODE`), so an edit + there re-keys that phase alone: the assembled gate, which every later + spine stage waits on, re-keys on none of the transferred phase's (the + re-review of microcosm#1420, E; microcosm#931 ruling 6). """ ref = "uk.spine-gates@1" @@ -369,21 +459,34 @@ def __init__(self, *, gates: GatesManifest, engine: object, engine_identity: str self.engine_identity = engine_identity def implementation_hash(self) -> str: + import microcosm.calibrate.geography_constants as geography_constants from microcosm.graph.canonical import canonical_json + from .. import gates, stage_evidence + from . import ( + council_tax_constants, + graph_kernels, + national_frame, + receipt_counts, + ) from .solve_scope import uk_solve_fingerprint # The spine stages after a gate wait on its verdict, so its key is # part of theirs. No spine gate reads a gate-only register, so it keys # on the solve fingerprint and a sign-off between two release builds - # leaves an in-graph spine a cache hit (microcosm#1301). The stage - # checks live in stage_health, so it keys the gate too (the independent - # review of microcosm#1420, finding 5), with stochastic_assignment, - # where the binomial band of its latent-attribute checks now lives - # (microcosm#931). Only the transferred phase reads the energy level, - # through energy_pricing, so that phase alone carries its hash, as a - # node param: an energy change re-keys it, not the assembled gate - # every later spine stage waits on (the re-review, E). + # leaves an in-graph spine a cache hit (microcosm#1301). The code is + # what both phases compute through (microcosm#931 ruling 6): this + # kernel's run, the frame it rebuilds (graph_kernels' _minimal_frame, + # national_frame) and the evidence it decodes (stage_evidence); the + # battery and the gate registry (gate_battery, battery_bindings, + # gates); the stage-health checks (stage_health, the independent + # review of microcosm#1420, finding 5), with the receipt bands of the + # take-up checks both phases declare (receipt_counts); and the enum + # gate's nation partition (council_tax_constants, the region tier). + # What only one phase's gates compute through is that phase's own + # code, a node param (UK_SPINE_GATE_PHASE_CODE): the energy level, + # for one, re-keys the transferred gate, not the assembled gate every + # later spine stage waits on (the re-review, E). return hashlib.sha256( canonical_json( { @@ -392,7 +495,13 @@ def implementation_hash(self) -> str: gate_battery, battery_bindings, stage_health, - stochastic_assignment, + gates, + stage_evidence, + graph_kernels, + national_frame, + receipt_counts, + council_tax_constants, + geography_constants, ), "country_resources": uk_solve_fingerprint(), } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 08f36ad832..147b42bdf5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -93,7 +93,7 @@ national_doctrine, ) from . import support_floor as support_floor_rules -from .battery_bindings import UK_GATE_REGISTRY +from .battery_bindings import UK_GATE_REGISTRY, uk_gate_registry_modules from .calibration_run import ( UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS, uk_local_gate_scope_exclusions, @@ -1027,16 +1027,17 @@ def implementation_hash(self) -> str: weighted_integrity, ) - # Outside the gate registry's evaluators (held for a ruling; see - # test_uk_graph's _KERNEL_STOPS) the battery also computes through - # the shared gates' evaluators (gates), the solved result and - # registry it decodes (microcosm.calibrate's artifacts, matrix, - # registry, target and hierarchy, graph_targets), the frame it - # rebuilds (graph_population, national_frame), the manifest's - # effective sample size (calibrate.solve), the admin totals' derived - # anchors (etb_services), the CGT projection (cgt_projection, - # donor_uprating) and the canonical encoding of its artifacts - # (microcosm#931). + # The battery evaluates whichever gates its node's gate scope names, + # so it keys on everything the gate registry's bindings compute + # through (uk_gate_registry_modules; microcosm#931 ruling 6). Its own + # code also computes through the shared gates' evaluators (gates), + # the solved result and registry it decodes (microcosm.calibrate's + # artifacts, matrix, registry, target and hierarchy, graph_targets), + # the frame it rebuilds (graph_population, national_frame), the + # manifest's effective sample size (calibrate.solve), the admin + # totals' derived anchors (etb_services), the CGT projection + # (cgt_projection, donor_uprating) and the canonical encoding of its + # artifacts (microcosm#931). return hashlib.sha256( canonical_json( { @@ -1066,6 +1067,7 @@ def implementation_hash(self) -> str: cgt_projection, donor_uprating, canonical_json, + *uk_gate_registry_modules(), ), "country_resources": load_country_spec("uk").fingerprint, } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 8e18f8d313..71fff0ef86 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -12,12 +12,12 @@ import io import json import sys -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import asdict, replace from functools import cache from importlib import metadata from pathlib import Path -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import Any import numpy as np @@ -821,54 +821,173 @@ def _source_gate_evidence(context: KernelContext, engine): @cache def uk_terminal_only_code_hash() -> str: - """The source hash of the code only the calibrated gates run. + """The source hash of the code only the calibrated gates compute through. - The high-value homes evidence is computed on the calibrated pool alone. The source preflight runs the gate kernel's code and the dense solve keys - on the preflight, so this code keys the calibrated node through its - parameters, not the kernel's hash (microcosm#1306 review). An edit to - ``high_value_home_gates`` re-keys the calibrated gates and what follows - them, never the solve. ``high_value_homes`` also feeds the target - kernels, which hash it, so an edit to it still re-solves: a disclosure fix - between two builds that must reuse the solve belongs in - ``high_value_home_gates`` alone. - - The calibrated node (``uk.full.gates.calibrated``) also publishes its - ``calibration_diagnostics`` output as schema 9: - ``catalog_calibration_diagnostics`` with the UK label choices (which read - ``UK_GEOGRAPHY_ID_TO_LABEL``), dumped from ``CalibrationDiagnosticsV9``. - Those modules decide the published labels and bytes, so an edit to them - re-keys the calibrated node the same way. ``geography_constants``, like - ``high_value_homes``, also feeds the target kernels, which hash it - (microcosm#1492), so an edit to it still re-solves; an edit to the other - three never does. + on the preflight, so the code only the terminal phase computes through + keys the calibrated node through its ``terminal_code`` parameter, not the + kernel's hash (microcosm#1306 review; microcosm#931 ruling 6): an edit to + it re-keys the calibrated gates and what follows them, never the solve. + That is the terminal branch of the kernel's run, and what the terminal + gates compute through beyond the preflight's: + + - the high-value homes evidence, on the calibrated pool alone + (``high_value_home_gates``, with the disclosure floor it reads from + ``receipt_counts`` and the tenure categories behind its owner mask, + ``tenure_constants``). ``high_value_homes`` itself is in the kernel's + hash, since the preflight's target-surface evidence reads its metric + names, and the target kernels hash it too, so an edit to it re-solves: + a disclosure fix between two builds that must reuse the solve belongs + in ``high_value_home_gates`` alone. ``calibrate.geography_constants``, + which the preflight's ledger parity reads, is in the kernel's hash too; + - the calibration diagnostics the calibrated node publishes as schema 9: + ``uk_runtime.diagnostics``, with its consumption drift + (``domestic_energy_targets``, ``energy_pricing``, ``etb_vat``, + ``lcfs_consumption``) and label choices (``local_authority_input``, + ``calibrate.hierarchy_labels``); ``microcosm.calibrate``'s + diagnostics, target loss attribution and solve; and + ``microcosm.diagnostics``' catalogs and schema; + - the solved problem and result it decodes (``microcosm.calibrate``'s + artifacts, matrix and monetary binding), the frame it rebuilds + (``graph_population``), the area-support summary (``local_rowwise``), + the support floors, admin anchors and CGT projection it builds the + gate context with (``support_floor``, ``etb_services``, + ``cgt_projection``, ``donor_uprating``), the input-mass reference + (``weighted_integrity``) and the fit-weight records it rehydrates + (``release_certification``); + - the terminal gates' evaluators: the input-mass parity + (``input_mass``), the CGT projection fence's conditioning facts + (``hmrc_capital_gains``, read through the vendored-fact reader, the + chronicle feed and its schema epochs), the take-up signal + (``frs_take_up``, ``take_up_contract``, ``dfe_spring_term``), the + column implication's property shares (``uc_capital_coherence``), the + nonnegative columns (``source_runtime``) and the geography ladder + (``rowwise_geography``). Hashed once per process, as the code was imported: an edit to the files while a build runs cannot fail the calibrated gates after the solve. """ - import microcosm.calibrate.geography_constants as geography_constants + import microcosm.calibrate._target_loss_attribution as target_loss_attribution + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.diagnostics as calibrate_diagnostics import microcosm.calibrate.hierarchy_labels as hierarchy_labels + import microcosm.calibrate.matrix as calibrate_matrix + import microcosm.calibrate.monetary_binding as monetary_binding + import microcosm.calibrate.solve as calibrate_solve import microcosm.diagnostics.catalogs as diagnostics_catalogs import microcosm.diagnostics.schema as diagnostics_schema - from . import high_value_home_gates, high_value_homes + from .. import chronicle_epoch, input_mass + from . import ( + cgt_projection, + chronicle_feed, + dfe_spring_term, + diagnostics, + domestic_energy_targets, + donor_uprating, + energy_pricing, + etb_services, + etb_vat, + frs_take_up, + graph_population, + high_value_home_gates, + hmrc_capital_gains, + lcfs_consumption, + ledger_fact_vendoring, + local_authority_input, + local_rowwise, + receipt_counts, + release_certification, + rowwise_geography, + source_runtime, + support_floor, + take_up_contract, + tenure_constants, + uc_capital_coherence, + weighted_integrity, + ) return source_hash( high_value_home_gates, - high_value_homes, diagnostics_catalogs, diagnostics_schema, hierarchy_labels, - geography_constants, + diagnostics, + calibrate_diagnostics, + target_loss_attribution, + calibrate_artifacts, + calibrate_matrix, + monetary_binding, + calibrate_solve, + graph_population, + local_rowwise, + support_floor, + rowwise_geography, + release_certification, + weighted_integrity, + input_mass, + etb_services, + etb_vat, + domestic_energy_targets, + energy_pricing, + lcfs_consumption, + cgt_projection, + donor_uprating, + hmrc_capital_gains, + frs_take_up, + take_up_contract, + dfe_spring_term, + receipt_counts, + uc_capital_coherence, + tenure_constants, + source_runtime, + local_authority_input, + ledger_fact_vendoring, + chronicle_feed, + chronicle_epoch, ) +@cache +def uk_preflight_only_code_hash() -> str: + """The source hash of the code only the source preflight computes + through: the shared ledger's registry parity report, behind the ledger + compile parity gates only the preflight declares + (``microcosm.build.ledger_targets``). + + The preflight's node carries it as its ``preflight_code`` parameter, so + an edit to it re-keys the preflight and the solve after it, and the + calibrated gates only through them (microcosm#931 ruling 6). Hashed once + per process, as the calibrated gates' own code is. + """ + from .. import ledger_targets as shared_ledger_targets + + return source_hash(shared_ledger_targets) + + +#: Each full gate phase's own code: the hash of what only that phase computes +#: through, which its node carries as its ``_code`` parameter and the +#: kernel checks before it evaluates anything. The code both phases compute +#: through is the gate kernels' shared hash (microcosm#931 ruling 6); +#: test_uk_graph derives each phase's share from the gates it declares and +#: the branch of the kernel's run it takes. +UK_FULL_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( + { + "preflight": uk_preflight_only_code_hash, + "terminal": uk_terminal_only_code_hash, + } +) + + class UKFullGateKernel(KernelBase): """The calibrated gate battery: every terminal gate, on the final weights. Its terminal gates read the gate-only registers, so it keys on the full country fingerprint and a sign-off re-keys it. The source preflight runs - under :class:`UKFullPreflightGateKernel` (microcosm#1301). + under :class:`UKFullPreflightGateKernel` (microcosm#1301). Both phases + key on the code both compute through, this kernel's hash, and each on + its own, which its node carries as ``_code`` and the kernel + checks (:data:`UK_FULL_GATE_PHASE_CODE`; microcosm#931 ruling 6). """ ref = "uk.full-gates@1" @@ -893,17 +1012,42 @@ def _country_resources(self) -> str: return load_country_spec("uk").fingerprint def implementation_hash(self) -> str: - # The gate evaluators (``gates``) and the register loaders - # (``terminal_gates``) decide the report as much as the bindings do, - # so an edit to either re-keys the gate nodes (microcosm#1301). - from .. import gate_battery, gates + # The code both phases compute through (microcosm#931 ruling 6), as + # test_uk_graph derives it from the gates each declares and the branch + # of run each takes: this kernel's run, with the manifests, + # enforcement and source registries it reads and the reports it + # encodes; the battery, the gate registry and the shared gates' + # evaluators; and what both phases' gates compute through, among it + # the release input coverage and the frame surface it reads, the + # target surfaces and their expected sets, and the register loaders + # (terminal_gates). What only one phase computes through is that + # phase's own code (UK_FULL_GATE_PHASE_CODE), so the preflight, and + # the dense solve keyed on it, re-key on none of the terminal phase's. + import microcosm.calibrate.geography_constants as geography_constants + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.registry as calibrate_registry + import microcosm.calibrate.target as calibrate_target + import microcosm.frame.materialize as frame_materialize + import microcosm.graph.canonical as canonical_encoder + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import gate_battery, gates, stage_evidence from . import ( battery_bindings, - diagnostics, + calibration_run, + council_tax_constants, full_gates, - local_rowwise, + geography_ladder, + graph_evidence, + graph_targets, + high_value_homes, + ledger_targets, + local_targets, + national_frame, + parity_reference, + release_identity, + release_input_coverage, terminal_gates, - weighted_integrity, ) return hashlib.sha256( @@ -915,10 +1059,27 @@ def implementation_hash(self) -> str: gates, battery_bindings, full_gates, - local_rowwise, terminal_gates, - weighted_integrity, - diagnostics, + stage_evidence, + graph_evidence, + graph_targets, + calibration_run, + release_input_coverage, + national_frame, + uk_engine_adapter, + frame_materialize, + release_identity, + parity_reference, + ledger_targets, + local_targets, + high_value_homes, + geography_ladder, + council_tax_constants, + geography_constants, + calibrate_registry, + calibrate_target, + calibrate_hierarchy, + canonical_encoder, ), "country_resources": self._country_resources(), } @@ -959,12 +1120,9 @@ def run(self, context: KernelContext) -> KernelResult: raise ValueError( "UK full gate engine identity differs from its declared binding." ) - if ( - context.params["phase"] == "terminal" - and context.params.get("terminal_code") != uk_terminal_only_code_hash() - ): + if context.params.get(f"{phase}_code") != UK_FULL_GATE_PHASE_CODE[phase](): raise ValueError( - "UK full gate terminal-only code differs from its declared binding." + f"UK full gate {phase}-only code differs from its declared binding." ) selection = json.loads(context.artifacts["selection"].payload)["receipt"] gates = uk_full_gate_manifest(selection) @@ -1197,7 +1355,9 @@ class UKFullPreflightGateKernel(UKFullGateKernel): preflight's key is part of the solve's. Its gates read no gate-only register (``solve_scope``), so it keys on the solve fingerprint: a sign-off between the two release builds leaves it, and so the solve, a - cache hit (microcosm#1301). The code it runs is the calibrated kernel's. + cache hit (microcosm#1301). The code it runs is the calibrated kernel's, + and it keys on that kernel's hash and on its own code, which its node + carries as ``preflight_code`` (microcosm#931 ruling 6). """ ref = "uk.full-gates.preflight@1" @@ -1306,7 +1466,14 @@ def append_uk_full_gate_nodes( "uk.full.gates.preflight", UKFullPreflightGateKernel.ref, population=input_population, - params={**params, "phase": "preflight"}, + # The preflight keys on its own code, as the calibrated node keys on + # its own, so an edit to either phase's alone re-keys only that phase + # and what follows it (microcosm#931 ruling 6). + params={ + **params, + "phase": "preflight", + "preflight_code": UK_FULL_GATE_PHASE_CODE["preflight"](), + }, artifact_inputs=common, artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), description="Validate complete source/reference registries and spine stage ownership before dense calibration.", @@ -1356,7 +1523,7 @@ def append_uk_full_gate_nodes( params={ **gate_params, "phase": "terminal", - "terminal_code": uk_terminal_only_code_hash(), + "terminal_code": UK_FULL_GATE_PHASE_CODE["terminal"](), }, artifact_inputs=( *common, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index dba6eb2363..84120f3078 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -3,7 +3,7 @@ # ruff: noqa: F403, F405 import functools import hashlib -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import MappingProxyType from typing import NamedTuple @@ -1202,6 +1202,15 @@ def changed_module_bytes(path): "the executor fills, and every node's inputs pass through it, where " "H2 catches a changed value (graph.kernel's core reason)" ), + ("microcosm.graph.kernel", "source_hash"): ( + "hashes the source files of the modules it is given. The full gates' " + "run recomputes its phase's own code through it only to refuse a node " + "whose _code parameter differs " + "(graph_terminal.UK_FULL_GATE_PHASE_CODE); that parameter is computed " + "through it too and node_key hashes it, so an edit that changes what " + "it returns changes the parameter and the check alike, and every key " + "(graph.kernel's core reason)" + ), ("microcosm.graph.kernel", "KernelResult"): ( "the frozen record a kernel returns its outputs in, which holds what " "the kernel puts in it and computes nothing; the executor validates " @@ -1279,12 +1288,23 @@ def changed_module_bytes(path): #: The same for the UK graph kernels: names a kernel reads from a module its #: key does not hash, each with why. _KERNEL_UNBOUND_NAMES: dict[tuple[str, str], str] = { - ("microcosm.build.uk_runtime.geography_sources", name): ( - "the LAD23 names lookup's pinned URL and digest: local_authority_input " - "refuses an input whose declared source differs from them, and reads " - "nothing through them" - ) - for name in ("LAD23_NAMES_SHA256", "LAD23_NAMES_URL") + **{ + ("microcosm.build.uk_runtime.geography_sources", name): ( + "the LAD23 names lookup's pinned URL and digest: " + "local_authority_input refuses an input whose declared source " + "differs from them, and reads nothing through them" + ) + for name in ("LAD23_NAMES_SHA256", "LAD23_NAMES_URL") + }, + ("microcosm.build.uk_runtime.rowwise_dataset", "ladder_clone_index_column"): ( + "names the geographic clone index columns, which only the dense expand " + "writes, and uk.full.expand@1 hashes rowwise_dataset. The calibrated " + "gates' support-family fold reads the household's on the expanded " + "frame, so their key chains on the expand's; the national gates fold " + "the national frame, which no ladder expands, and the fold keys on the " + "column only where the table carries it " + "(diagnostics.uk_support_family_weights)" + ), } #: Where the kernel rule stops, each with why a kernel that names the #: definition need not hash what lies beyond it; @@ -1315,36 +1335,87 @@ def changed_module_bytes(path): "_minimal_frame rebuilds the SPI support allocation's mass record " "through it for the hmrc_spi_income_spine stage alone, inside the " "branch on that stage's name, and that stage's node hashes everything " - "it computes through" + "it computes through; the spine gates, which rebuild their frame " + "through _minimal_frame too, name no stage" ), ("microcosm.build.uk_runtime.battery_bindings", "UK_GATE_REGISTRY"): ( "maps each UK gate to its evaluator. The bound spine builds a battery " "run through it only for the checkpoint's gate digests, which read the " "manifest, and the run reads the registry only to refuse a parameter " - "its binding does not declare. A kernel that evaluates gates through " - "it is held (_HELD_KERNELS, _HELD_KERNEL_STOPS)" + "its binding does not declare. The kernels that evaluate gates through " + "it are held past it (microcosm#931 ruling 6): each gate battery " + "phase to what the gates it declares compute through " + "(_GATE_BATTERY_KERNELS), and the national gates to its whole reach " + "(_WHOLE_REGISTRY_KERNELS)" ), } -#: Kernels the rule holds at one listed stop only, for a ruling -#: (microcosm#931), each with why. Past the stop the rule holds them as any -#: other kernel. test_every_hold_still_guards_a_module_its_kernel_leaves_out -#: checks that the walk past each stop still meets a module the kernel's hash -#: leaves out. -_HELD_KERNEL_STOPS: dict[tuple[str, tuple[str, str]], str] = { - ( - "uk.full.national-gates@1", - ("microcosm.build.uk_runtime.battery_bindings", "UK_GATE_REGISTRY"), - ): ( - "evaluates the gates its manifest declares through the registry, and " - "how a gate kernel's key binds what each evaluator computes through is " - "held with the gate batteries (_HELD_KERNELS)" +#: The gate registry: every kernel's walk stops there (``_KERNEL_STOPS``) but +#: the walks of the kernels that evaluate gates through it. +_GATE_REGISTRY = ("microcosm.build.uk_runtime.battery_bindings", "UK_GATE_REGISTRY") +#: The gate batteries' kernels, each with its battery (``_gate_batteries``). +#: A battery's phases run one kernel's code, whose hash they share, and each +#: evaluates the gates its manifest declares for it. The rule holds each phase +#: to what it computes through (``_battery_phase_walk``): its key, that hash +#: with the phase's own code, which its node carries as ``_code``, +#: covers it and carries nothing only another phase computes through, so an +#: edit only one phase computes through re-keys that phase alone +#: (microcosm#931 ruling 6; test_each_gate_battery_phase_keys_on_its_own_reach). +_GATE_BATTERY_KERNELS: dict[str, str] = { + "uk.spine-gates@1": "spine", + "uk.full-gates.preflight@1": "full", + "uk.full-gates@1": "full", +} +#: Kernels that can evaluate any gate the registry binds, each with why. The +#: rule walks them past the registry's stop, so their keys bind everything +#: its bindings compute through (microcosm#931 ruling 6). +_WHOLE_REGISTRY_KERNELS: dict[str, str] = { + "uk.full.national-gates@1": ( + "evaluates one phase, whose gates its node's gate_scope parameter " + "names, so no phase of its can be keyed on another's code; it hashes " + "every module the registry's bindings compute through " + "(battery_bindings.UK_GATE_REGISTRY_MODULES)" + ), +} +#: The stage-health dispatcher, which runs the one check its gate entry names +#: in its ``check`` parameter, each in its own ``if check == ...`` branch. A +#: battery phase follows only the branches of the checks it declares, and +#: what the dispatcher runs outside them +#: (test_the_battery_model_reads_the_declared_dispatch checks the shape). +_STAGE_HEALTH_DISPATCH = ( + "microcosm.build.uk_runtime.stage_health", + "uk_stage_health_gate", +) +#: Functions that hand a battery's phases evidence artifacts as one literal +#: mapping by artifact name, each with why a phase computes through an entry +#: only where one of its declared gates reads that artifact +#: (``_artifacts_read``). +_EVIDENCE_ARTIFACT_TABLES: dict[tuple[str, str], str] = { + ("microcosm.build.uk_runtime.graph_evidence", "uk_spine_gate_artifacts"): ( + "hands every spine phase, and the calibrated gates' context, the rules " + "engine and two frame-only enum domains. An enum gate reads only the " + "domain its one declared column names, so the CGT asset type domain, " + "which only a transferred-phase gate checks, is the transferred " + "phase's alone, and the ONS household type domain the assembled " + "phase's; the rules engine is the kernel's own, no first-party code" + ), +} +#: Artifacts a gate's evaluator reads beyond those its binding declares +#: (``required_artifacts``), by gate: what it reads, from an entry's +#: parameters, and why. +_UNDECLARED_ARTIFACT_READS: dict[str, tuple[Callable, str]] = { + "enum_domain": ( + lambda parameters: frozenset( + {f"{tuple(parameters['columns'])[0]}_enum_domain", "rules_engine"} + ), + "battery_bindings._evaluate_enum_domain reads the domain its one " + "declared column names, and resolves it from the rules engine where " + "that artifact is absent", ), } #: Kernels the rule does not yet hold, each with why, all held for a ruling -#: (microcosm#931): the gate batteries, whose phases share one key but declare -#: different gates, and the target kernels, whose keys microcosm#1497's -#: import record holds. Until the ruling, a held kernel can replay a stale -#: output after an edit to a module it computes through but does not hash; +#: (microcosm#931): the target kernels, whose keys microcosm#1497's import +#: record holds. Until the ruling, a held kernel can replay a stale output +#: after an edit to a module it computes through but does not hash; #: test_every_hold_still_guards_a_module_its_kernel_leaves_out checks that #: each still does, so a hold cannot outlive its need. _TARGET_RECORD_REASON = ( @@ -1355,37 +1426,15 @@ def changed_module_bytes(path): "import of each module bound" ) _HELD_KERNELS: dict[str, str] = { - "uk.spine-gates@1": ( - "the assembled and transferred spine phases share one key, and only " - "the transferred phase declares the gates that check the CGT asset " - "type domain and most of the stage-health checks. Binding what those " - "compute through into the shared key would re-key the assembled gate, " - "which every later spine stage waits on, as the re-review of " - "microcosm#1420 (E) kept the energy level out of it" - ), - "uk.full-gates.preflight@1": ( - "runs the calibrated battery's code, one class body and one code " - "hash, for the preflight phase only. The dense solve keys on it, so " - "binding the terminal phase's modules into the shared hash would " - "re-solve on any terminal-gate edit, which microcosm#1301 and #1306 " - "keyed out" - ), - "uk.full-gates@1": ( - "shares its code hash with the preflight, and its terminal-only " - "publication code keys it through its terminal_code parameter " - "(microcosm#1306)" - ), - **{ - ref: _TARGET_RECORD_REASON - for ref in ( - "uk.full.target_compilation@1", - "uk.full.target_selection@1", - "uk.full.measures@1", - "uk.full.problem@1", - "uk.full.national_targets@1", - "uk.full.national_problem@1", - ) - }, + ref: _TARGET_RECORD_REASON + for ref in ( + "uk.full.target_compilation@1", + "uk.full.target_selection@1", + "uk.full.measures@1", + "uk.full.problem@1", + "uk.full.national_targets@1", + "uk.full.national_problem@1", + ) } #: Packages that export names lazily, through a module ``__getattr__``, each #: with the table, of name to the submodule defining it, that ``__getattr__`` @@ -2505,10 +2554,13 @@ def _kernel_body(kernel_class: type) -> list[tuple[type, object, str]]: order that defines it, as Python resolves it, and a base's definition too where the one that wins calls ``super()``. ``implementation_hash`` is left out, since it computes the key: an edit to what it calls that changes what - it returns changes the key. So are the listed refusal-only methods.""" + it returns changes the key. So is a method only the key computation calls + (``_key_only_methods``), such as the full gates' ``_country_resources``, + and so are the listed refusal-only methods.""" chosen: dict[str, object | None] = {} body = [] + keyed = [] for cls in _kernel_classes(kernel_class): definitions = _module_definitions(cls.__module__, _HELPER_SCOPE).definitions for statement in definitions[cls.__name__].body: @@ -2517,16 +2569,71 @@ def _kernel_body(kernel_class: type) -> list[tuple[type, object, str]]: previous = chosen[name] if previous is not None and _calls_super(previous): chosen[name] = statement - body.append((cls, statement, name)) + (keyed if name == "implementation_hash" else body).append( + (cls, statement, name) + ) continue - if name == "implementation_hash" or ( - (cls.__module__, f"{cls.__name__}.{name}") in _REFUSAL_ONLY_HELPERS - ): + if (cls.__module__, f"{cls.__name__}.{name}") in _REFUSAL_ONLY_HELPERS: chosen[name] = None continue chosen[name] = statement - body.append((cls, statement, name)) - return body + (keyed if name == "implementation_hash" else body).append( + (cls, statement, name) + ) + key_only = _key_only_methods(body, keyed) + return [item for item in body if item[2] not in key_only] + + +def _self_attributes(statement) -> set[str]: + """The attributes ``statement`` reads on ``self``, ``cls`` or + ``type(self)``.""" + import ast + + found = set() + for sub in ast.walk(statement): + if not isinstance(sub, ast.Attribute): + continue + value = sub.value + if (isinstance(value, ast.Name) and value.id in {"self", "cls"}) or ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Name) + and value.func.id == "type" + ): + found.add(sub.attr) + return found + + +def _key_only_methods(body, keyed) -> frozenset[str]: + """The methods of a kernel's ``body`` that only its key computation calls: + named on ``self``, ``cls`` or ``type(self)`` by its ``keyed`` statements + (``implementation_hash``) or by another such method, and by nothing else + the kernel runs. They compute the key as the hash does.""" + import ast + + methods = { + name + for _, statement, name in body + if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef) + } + reads: dict[str, set[str]] = {} + for _, statement, name in body: + reads.setdefault(name, set()).update(_self_attributes(statement)) + key = set().union(*(_self_attributes(statement) for _, statement, _ in keyed)) + key_only: set[str] = set() + while True: + found = { + method + for method in methods - key_only + if method in key.union(*(reads[item] for item in key_only)) + and not any( + method in names + for other, names in reads.items() + if other != method and other not in key_only + ) + } + if not found: + return frozenset(key_only) + key_only |= found def _kernel_walk( @@ -2557,11 +2664,11 @@ def _kernel_walk( return _KernelWalk(walk.reach, walk.stops, walk.core, met, core_names) -def _hashed_modules(kernel) -> frozenset[str]: - """Every module ``kernel.implementation_hash()`` hashes, recorded at each - ``source_hash`` it reaches, in whichever module calls it: every loaded - module's ``source_hash`` name stands in for the recorder during one call, - after a first call has imported what the hash imports.""" +def _hashed_by(compute: Callable[[], object]) -> frozenset[str]: + """Every module ``compute()`` hashes, recorded at each ``source_hash`` it + reaches, in whichever module calls it: every loaded module's + ``source_hash`` name stands in for the recorder during one call, after a + first call has imported what ``compute`` imports.""" import inspect import sys @@ -2576,7 +2683,7 @@ def record(*objects, **dependencies): recorded.extend(objects) return original(*objects, **dependencies) - kernel.implementation_hash() + compute() patched = [ module for module in list(sys.modules.values()) @@ -2586,7 +2693,7 @@ def record(*objects, **dependencies): for module in patched: module.source_hash = record try: - kernel.implementation_hash() + compute() finally: for module in patched: module.source_hash = original @@ -2596,6 +2703,12 @@ def record(*objects, **dependencies): ) +def _hashed_modules(kernel) -> frozenset[str]: + """Every module ``kernel.implementation_hash()`` hashes (``_hashed_by``).""" + + return _hashed_by(kernel.implementation_hash) + + @functools.cache def _uk_graph_kernels() -> MappingProxyType: """Every kernel the UK registries register, by ref: the spine's (the @@ -2635,6 +2748,343 @@ def _uk_graph_kernels() -> MappingProxyType: return MappingProxyType({ref: registry.get(ref) for ref in registry.refs()}) +@functools.cache +def _registry_bindings() -> MappingProxyType: + """Each gate the UK registry binds, to the module and the expression of + its binding, read from the registry's literal mapping as Python builds it: + a later key overrides an earlier one, and a ``**`` entry is the mapping it + unpacks.""" + import ast + + def entries(module: str, name: str) -> dict[str, tuple[str, object]]: + value = _module_definitions(module, _HELPER_SCOPE).definitions[name].value + assert isinstance(value, ast.Dict), (module, name) + found: dict[str, tuple[str, object]] = {} + for key, item in zip(value.keys, value.values, strict=True): + if key is None: + assert isinstance(item, ast.Name), (module, name) + found.update(entries(*_resolve(module, item.id, _HELPER_SCOPE))) + else: + assert isinstance(key, ast.Constant), (module, name) + found[key.value] = (module, item) + return found + + return MappingProxyType(entries(*_GATE_REGISTRY)) + + +def _checked_name(statement) -> str | None: + """The check an ``if check == "":`` statement with no ``else`` runs, + or None.""" + import ast + + test = getattr(statement, "test", None) + if ( + isinstance(statement, ast.If) + and not statement.orelse + and isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and test.left.id == "check" + and len(test.ops) == 1 + and isinstance(test.ops[0], ast.Eq) + and isinstance(test.comparators[0], ast.Constant) + and isinstance(test.comparators[0].value, str) + ): + return test.comparators[0].value + return None + + +@functools.cache +def _stage_health_dispatch() -> tuple[MappingProxyType, tuple]: + """The stage-health dispatcher's branches, by the check each runs, and its + statements outside them (``_STAGE_HEALTH_DISPATCH``).""" + + module, name = _STAGE_HEALTH_DISPATCH + node = _module_definitions(module, _HELPER_SCOPE).definitions[name] + branches: dict[str, tuple] = {} + common = [] + for statement in node.body: + check = _checked_name(statement) + if check is None: + common.append(statement) + else: + assert check not in branches, check + branches[check] = tuple(statement.body) + return MappingProxyType(branches), tuple(common) + + +def _phase_test(test) -> str | None: + """The phase an ``if`` test ``phase == ""`` selects, or None.""" + import ast + + if ( + isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and test.left.id == "phase" + and len(test.ops) == 1 + and isinstance(test.ops[0], ast.Eq) + and isinstance(test.comparators[0], ast.Constant) + and isinstance(test.comparators[0].value, str) + ): + return test.comparators[0].value + return None + + +def _phase_branch(statement, phase: str): + """``statement`` as it runs for ``phase``: each ``if phase == ""`` + keeps only the branch ``phase`` takes, down its ``elif`` chain.""" + import ast + import copy + + class _Prune(ast.NodeTransformer): + def visit_If(self, node): + selected = _phase_test(node.test) + if selected is None: + self.generic_visit(node) + return node + kept = [] + for item in node.body if selected == phase else node.orelse: + visited = self.visit(item) + kept.extend(visited if isinstance(visited, list) else [visited]) + return kept or [ast.Pass()] + + return _Prune().visit(copy.deepcopy(statement)) + + +@functools.cache +def _artifact_table(table: tuple[str, str]) -> MappingProxyType: + """An evidence-artifact table's entries (``_EVIDENCE_ARTIFACT_TABLES``): + each artifact's name to the expression of its value. The function returns + one literal mapping with string keys, after at most a docstring.""" + import ast + + module, name = table + node = _module_definitions(module, _HELPER_SCOPE).definitions[name] + body = [ + statement + for statement in node.body + if not ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Constant) + ) + ] + assert len(body) == 1 and isinstance(body[0], ast.Return), table + mapping = body[0].value + assert isinstance(mapping, ast.Dict), table + entries = {} + for key, value in zip(mapping.keys, mapping.values, strict=True): + assert isinstance(key, ast.Constant) and isinstance(key.value, str), table + entries[key.value] = value + return MappingProxyType(entries) + + +def _artifacts_read(entry) -> frozenset[str]: + """The evidence artifacts a gate entry reads: those its binding declares + for its parameters (``required_artifacts``, an absent one of which makes + it ``evidence_absent``), and those its evaluator reads beyond them + (``_UNDECLARED_ARTIFACT_READS``).""" + from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY + + read = set(UK_GATE_REGISTRY[entry.gate].required_artifacts(entry.parameters)) + if entry.gate in _UNDECLARED_ARTIFACT_READS: + read |= _UNDECLARED_ARTIFACT_READS[entry.gate][0](entry.parameters) + return frozenset(read) + + +def _battery_phase_walk(kernel_class: type, gates, phase: str) -> _KernelWalk: + """What one phase of a gate battery computes through (microcosm#931 + ruling 6): the kernel rule's walk from what the kernel runs, each ``if + phase == ...`` taking the branch ``phase`` takes (``_phase_branch``), and + from the binding of each gate the manifest ``gates`` declares for + ``phase`` (``_registry_bindings``), the registry's stop aside. An entry + the manifest excuses as not applicable, or whose gate has no binding, + evaluates nothing. The stage-health + dispatcher is followed only into the branches of the checks the phase + declares (``_STAGE_HEALTH_DISPATCH``), and an evidence-artifact table the + walk meets only into the entries the phase's gates read + (``_EVIDENCE_ARTIFACT_TABLES``, ``_artifacts_read``). ``met`` holds the + listed kernel stops it meets.""" + + entries = [ + entry + for entry in gates.gates + if entry.phase == phase and entry.not_applicable is None + ] + roots = {(cls.__module__, cls.__name__) for cls in _kernel_classes(kernel_class)} + seeds = [ + (found, (cls.__module__, f"{cls.__name__}.{name}")) + for cls, statement, name in _kernel_body(kernel_class) + for found in _named( + cls.__module__, _phase_branch(statement, phase), _HELPER_SCOPE + ) + ] + bindings = _registry_bindings() + # A gate with no binding evaluates nothing: its entry is evidence_absent. + for gate in sorted({entry.gate for entry in entries} & set(bindings)): + module, value = bindings[gate] + seeds.extend( + (found, (module, f"{_GATE_REGISTRY[1]}[{gate!r}]")) + for found in _named(module, value, _HELPER_SCOPE) + ) + checks = sorted( + { + str(entry.parameters["check"]) + for entry in entries + if entry.gate == "stage_health" + } + ) + if checks: + branches, common = _stage_health_dispatch() + roots.add(_STAGE_HEALTH_DISPATCH) + seeds.extend( + (found, _STAGE_HEALTH_DISPATCH) + for statement in ( + *common, + *(item for check in checks for item in branches.get(check, ())), + ) + for found in _named(_STAGE_HEALTH_DISPATCH[0], statement, _HELPER_SCOPE) + ) + read = frozenset().union(*map(_artifacts_read, entries)) + halts = { + **_KERNEL_STOPS, + _STAGE_HEALTH_DISPATCH: "the stage-health dispatch", + **_EVIDENCE_ARTIFACT_TABLES, + } + followed: set[tuple[str, str]] = set() + while True: + walk, met, core_names = _walk(roots, seeds, _HELPER_SCOPE, halts=halts) + tables = (set(_EVIDENCE_ARTIFACT_TABLES) & set(met)) - followed + if not tables: + break + for table in sorted(tables): + roots.add(table) + seeds.extend( + (found, table) + for artifact, value in _artifact_table(table).items() + if artifact in read + for found in _named(table[0], value, _HELPER_SCOPE) + ) + followed |= tables + return _KernelWalk( + walk.reach, + walk.stops, + walk.core, + {item: namers for item, namers in met.items() if item in _KERNEL_STOPS}, + core_names, + ) + + +class _Battery(NamedTuple): + """A gate battery: the manifest its kernels evaluate, the kernel each + phase runs under, and the table of each phase's own code.""" + + gates: object + phases: Mapping[str, str] + codes: Mapping[str, Callable[[], str]] + + +@functools.cache +def _gate_batteries() -> MappingProxyType: + """The UK gate batteries (``_GATE_BATTERY_KERNELS``): the spine gates, + whose one kernel runs both spine phases of the manifest it is bound to, + and the full build's preflight and calibrated gates, each running the one + phase it declares of the full build's manifest.""" + from microcosm.build.uk_runtime import graph_evidence, graph_terminal + from microcosm.build.uk_runtime.full_gates import uk_full_gate_manifest + + kernels = _uk_graph_kernels() + spine = kernels["uk.spine-gates@1"] + full = ("uk.full-gates.preflight@1", "uk.full-gates@1") + return MappingProxyType( + { + "spine": _Battery( + spine.gates, + MappingProxyType({phase: spine.ref for phase in spine.gates.phases}), + graph_evidence.UK_SPINE_GATE_PHASE_CODE, + ), + "full": _Battery( + uk_full_gate_manifest(), + MappingProxyType( + {phase: ref for ref in full for phase in kernels[ref].phases} + ), + graph_terminal.UK_FULL_GATE_PHASE_CODE, + ), + } + ) + + +def _phase_code_modules(battery: _Battery, phase: str) -> frozenset[str]: + """Every module a phase's own code hashes (``_hashed_by``), uncached.""" + + code = battery.codes.get(phase) + if code is None: + return frozenset() + return _hashed_by(getattr(code, "__wrapped__", code)) + + +class _KeyedWalk(NamedTuple): + """A walk the kernel rule holds a key to, and the modules that key + hashes.""" + + walk: _KernelWalk + key: frozenset[str] + + +@functools.cache +def _kernel_rule_walks() -> MappingProxyType: + """What the kernel rule holds each UK graph kernel to, by ref and label: + each walk with the modules its key hashes. An ordinary kernel has one, its + walk (``_kernel_walk``) against its implementation hash; one listed in + ``_WHOLE_REGISTRY_KERNELS`` walks past the gate registry's stop. A gate + battery's kernel has one per phase it runs, labelled ``" "`` + (``_battery_phase_walk``), against its hash with the phase's own code. A + class with several kernels (the spine's stage kernels) is walked once. + The held kernels (``_HELD_KERNELS``) are left out.""" + + kernels = _uk_graph_kernels() + batteries = _gate_batteries() + by_class: dict[type, list[tuple[str, object]]] = {} + for ref, kernel in sorted(kernels.items()): + by_class.setdefault(type(kernel), []).append((ref, kernel)) + rule = {} + for kernel_class, members in by_class.items(): + refs = {ref for ref, _ in members} + held = {ref in _HELD_KERNELS for ref in refs} + assert len(held) == 1, kernel_class + if held == {True}: + continue + if refs & set(_GATE_BATTERY_KERNELS): + assert refs <= set(_GATE_BATTERY_KERNELS), kernel_class + for ref, kernel in members: + battery = batteries[_GATE_BATTERY_KERNELS[ref]] + shared = _hashed_modules(kernel) + rule[ref] = { + f"{ref} {phase}": _KeyedWalk( + _battery_phase_walk(kernel_class, battery.gates, phase), + shared | _phase_code_modules(battery, phase), + ) + for phase, runner in battery.phases.items() + if runner == ref + } + continue + whole = {ref in _WHOLE_REGISTRY_KERNELS for ref in refs} + assert len(whole) == 1, kernel_class + walk = _kernel_walk( + kernel_class, + halts=( + { + item: why + for item, why in _KERNEL_STOPS.items() + if item != _GATE_REGISTRY + } + if whole == {True} + else None + ), + ) + for ref, kernel in members: + rule[ref] = {ref: _KeyedWalk(walk, _hashed_modules(kernel))} + return MappingProxyType(rule) + + def test_the_kernel_rule_sees_every_kernel_the_uk_runtime_defines() -> None: """The kernels the rule enumerates are every kernel class the UK runtime and the shared atomic geography define, so a new one cannot escape it, @@ -2669,50 +3119,52 @@ def test_the_kernel_rule_sees_every_kernel_the_uk_runtime_defines() -> None: defined.add(getattr(importlib.import_module(module), node.name)) kernels = _uk_graph_kernels() assert {type(kernel) for kernel in kernels.values()} == defined - assert set(_HELD_KERNELS) <= set(kernels) + listed = ( + set(_HELD_KERNELS), + set(_GATE_BATTERY_KERNELS), + set(_WHOLE_REGISTRY_KERNELS), + ) + for table in listed: + assert table <= set(kernels) + assert sum(map(len, listed)) == len(set().union(*listed)) def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: """The stage rule, for every other UK graph kernel (microcosm#931): each module holding code a kernel's own methods can run to compute a value is - in what its implementation hash really hashes, recorded at the - ``source_hash`` calls it makes, and every name the rule stops at outside - that hash is listed in ``_KERNEL_UNBOUND_NAMES`` with its reason. + in what its key really hashes, recorded at the ``source_hash`` calls it + makes, and every name the rule stops at outside that key is listed in + ``_KERNEL_UNBOUND_NAMES`` with its reason. The rule starts at what the kernel runs (``_kernel_body``: its methods by - method resolution order, without the hash itself and the listed - refusal-only methods), not at its class, and stops at the declaration, - refusal-only and core-module tables as for the stages, and at - ``_KERNEL_STOPS``. Where a class has several kernels (the spine's stage - kernels, one per stage), each one's key is held to it. The held kernels - (``_HELD_KERNELS``) are not. A name microcosm.graph exports lazily, - through its module ``__getattr__``, is followed to the runtime module - that defines it like any other (``_LAZY_EXPORT_TABLES``): the kernels' - capability and context records are listed core names, and the create - kernel's ``load_source``, which it calls only where no root transform is - bound, is a listed stop.""" + method resolution order, without the hash itself, the methods only the + hash calls and the listed refusal-only methods), not at its class, and + stops at the declaration, refusal-only and core-module tables as for the + stages, and at ``_KERNEL_STOPS``. Where a class has several kernels (the + spine's stage kernels, one per stage), each one's key is held to it. A + gate battery is held to it one phase at a time, each phase's key being + the battery's hash with the phase's own code (``_battery_phase_walk``), + and the national gates past the gate registry's stop + (``_WHOLE_REGISTRY_KERNELS``; microcosm#931 ruling 6). The held kernels + (``_HELD_KERNELS``) are not held to it. A name microcosm.graph exports + lazily, through its module ``__getattr__``, is followed to the runtime + module that defines it like any other (``_LAZY_EXPORT_TABLES``): the + kernels' capability and context records are listed core names, and the + create kernel's ``load_source``, which it calls only where no root + transform is bound, is a listed stop.""" from microcosm.build.uk_runtime import graph_population, rowwise_geography kernels = _uk_graph_kernels() - by_class: dict[type, list[tuple[str, object]]] = {} - for ref, kernel in sorted(kernels.items()): - by_class.setdefault(type(kernel), []).append((ref, kernel)) unhashed: dict[str, list[str]] = {} unlisted: set[tuple[str, str]] = set() met: set[tuple[str, str]] = set() - for kernel_class, members in by_class.items(): - held = {ref in _HELD_KERNELS for ref, _ in members} - assert len(held) == 1, kernel_class - if held == {True}: - continue - walk = _kernel_walk(kernel_class) - met |= set(walk.met) - for ref, kernel in members: - hashed = _hashed_modules(kernel) - missing = sorted(set(walk.reach) - hashed) + for walks in _kernel_rule_walks().values(): + for label, (walk, key) in walks.items(): + missing = sorted(set(walk.reach) - key) if missing: - unhashed[ref] = missing - unlisted.update(item for item in walk.stops if item[0] not in hashed) + unhashed[label] = missing + unlisted.update(item for item in walk.stops if item[0] not in key) + met |= set(walk.met) assert not unhashed, unhashed assert unlisted == set(_KERNEL_UNBOUND_NAMES), sorted( unlisted ^ set(_KERNEL_UNBOUND_NAMES) @@ -2736,25 +3188,246 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: assert rowwise_geography.__name__ in _hashed_modules(kernel), kernel.ref +def test_each_gate_battery_phase_keys_on_its_own_reach() -> None: + """Each phase of a gate battery keys on what it computes through and on + nothing only another phase computes through (microcosm#931 ruling 6). + + A battery's phases run one kernel's code, and that kernel's hash is the + code they share: it holds what every phase computes through and nothing + only some of them do. Each phase's own code, which its node carries as + ``_code``, holds exactly the rest of what that phase computes + through, and a phase that computes through nothing of its own carries + none. So an edit only the transferred spine phase computes through + re-keys neither the assembled gate nor the spine stages that wait on it, + and one only the terminal phase computes through re-keys neither the + preflight nor the dense solve keyed on it. What a phase computes through + is its walk (``_battery_phase_walk``), which the kernel rule's test holds + each phase's key to cover.""" + from microcosm.build.uk_runtime.graph_evidence import ( + UK_SPINE_GATE_PHASE_CODE, + add_uk_spine_gate_nodes, + ) + + kernels = _uk_graph_kernels() + rule = _kernel_rule_walks() + problems: dict[tuple[str, str], object] = {} + for name, battery in _gate_batteries().items(): + reaches = { + phase: frozenset(rule[ref][f"{ref} {phase}"].walk.reach) + for phase, ref in battery.phases.items() + } + every = frozenset.intersection(*reaches.values()) + some = frozenset.union(*reaches.values()) - every + (shared,) = {_hashed_modules(kernels[ref]) for ref in battery.phases.values()} + if every - shared: + problems[(name, "shared hash misses")] = sorted(every - shared) + if shared & some: + problems[(name, "shared hash holds a phase's own")] = sorted(shared & some) + assert set(battery.codes) <= set(battery.phases), name + for phase, reach in reaches.items(): + own = _phase_code_modules(battery, phase) + if own != reach - shared: + problems[(name, phase)] = { + "missing": sorted(reach - shared - own), + "extra": sorted(own - (reach - shared)), + } + if (phase in battery.codes) != bool(reach - shared): + problems[(name, f"{phase} code")] = sorted(reach - shared) + assert not problems, problems + # The spine gate nodes carry each phase's own code, and only their own; + # the full gate nodes' are held in test_uk_graph_terminal. + gated = add_uk_spine_gate_nodes( + uk_spine_graph(), spec=load_country_spec("uk"), engine_identity="test" + ) + for phase in _gate_batteries()["spine"].phases: + params = gated.node(f"spine.gates.{phase}").params + code = UK_SPINE_GATE_PHASE_CODE.get(phase) + assert {key for key in params if key.endswith("_code")} == ( + {f"{phase}_code"} if code is not None else set() + ), phase + if code is not None: + assert params[f"{phase}_code"] == code(), phase + + +def test_the_battery_model_reads_the_declared_dispatch() -> None: + """The per-phase walk models how a battery reaches each evaluator + (``_battery_phase_walk``), and each part of the model holds against the + code (microcosm#931 ruling 6): + + - the registry's literal mapping is the registry Python builds: the same + gates, each binding's evaluator the function its expression names; + - the stage-health dispatcher runs each check in its own branch, which + every check the UK spec declares has, and outside them only fails a + receipt for another stage or an unknown check; in the registry's reach + and the batteries', only the stage-health binding's evaluator names it; + - every ``if`` on ``phase`` in a battery kernel's code is one the walk + prunes, or only refuses; + - an evidence-artifact table's entry the model can skip is a domain an + enum gate reads by its one declared column, which nothing else in those + reaches reads by its key, or names no first-party definition; + - each battery's kernels are the listed ones, and run its phases.""" + import ast + + from microcosm.build.uk_runtime import battery_bindings + + # The registry. + bindings = _registry_bindings() + assert set(bindings) == set(battery_bindings.UK_GATE_REGISTRY) + for gate, (module, value) in bindings.items(): + binding = battery_bindings.UK_GATE_REGISTRY[gate] + keyword = "evaluator" if hasattr(binding, "evaluator") else "gate" + function = getattr(binding, keyword) + assert isinstance(value, ast.Call), gate + (named,) = [item.value for item in value.keywords if item.arg == keyword] + assert _named(module, named, _HELPER_SCOPE) == { + (function.__module__, function.__name__) + }, gate + # The stage-health dispatch. + branches, common = _stage_health_dispatch() + declared = { + str(entry.parameters["check"]) + for entry in load_country_spec("uk").gates.gates + if entry.gate == "stage_health" + } + assert declared <= set(branches), sorted(declared - set(branches)) + for check, body in branches.items(): + assert len(body) == 1 and isinstance(body[0], ast.Return), check + for statement in common: + if isinstance(statement, ast.Expr) and isinstance( + statement.value, ast.Constant + ): + continue + body = statement.body if isinstance(statement, ast.If) else [statement] + assert not getattr(statement, "orelse", None), ast.unparse(statement) + assert ( + len(body) == 1 + and isinstance(body[0], ast.Return) + and isinstance(body[0].value, ast.Call) + and ast.unparse(body[0].value.func) == "GateResult" + ), ast.unparse(statement) + # The registry's whole reach, with every battery phase's. + merged = _helper_walk(_GATE_REGISTRY) + for ref in _GATE_BATTERY_KERNELS: + for keyed in _kernel_rule_walks()[ref].values(): + merged = _merged(merged, _Walk(keyed.walk.reach, frozenset(), frozenset())) + reach = merged.reach + dispatchers = set() + domain_reads: dict[tuple[str, str], set[str]] = {} + for module, names in reach.items(): + definitions = _module_definitions(module, _HELPER_SCOPE).definitions + for name in names: + node = definitions[name] + if _STAGE_HEALTH_DISPATCH in _named(module, node, _HELPER_SCOPE): + dispatchers.add((module, name)) + for sub in ast.walk(node): + key = None + if isinstance(sub, ast.Subscript) and ast.unparse(sub.value).endswith( + "artifacts" + ): + key = sub.slice + elif ( + isinstance(sub, ast.Call) + and isinstance(sub.func, ast.Attribute) + and sub.func.attr == "get" + and ast.unparse(sub.func.value).endswith("artifacts") + and sub.args + ): + key = sub.args[0] + last = key.values[-1] if isinstance(key, ast.JoinedStr) else key + if isinstance(last, ast.Constant) and str(last.value).endswith( + "_enum_domain" + ): + domain_reads.setdefault((module, name), set()).add(ast.unparse(key)) + assert dispatchers == {(battery_bindings.__name__, "_evaluate_stage_health")} + enum = (battery_bindings.__name__, "_evaluate_enum_domain") + assert domain_reads == {enum: {"f'{column}_enum_domain'"}}, domain_reads + assigned = { + ast.unparse(sub) + for sub in ast.walk( + _module_definitions(enum[0], _HELPER_SCOPE).definitions[enum[1]] + ) + if isinstance(sub, ast.Assign) + } + assert { + "columns = tuple(parameters.get('columns', ()))", + "column = columns[0]", + } <= assigned + # The phase tests. + kernels = _uk_graph_kernels() + for ref in _GATE_BATTERY_KERNELS: + for _, statement, name in _kernel_body(type(kernels[ref])): + for sub in ast.walk(statement): + if not isinstance(sub, ast.If) or not any( + isinstance(item, ast.Name) and item.id == "phase" + for item in ast.walk(sub.test) + ): + continue + assert _phase_test(sub.test) is not None or all( + isinstance(item, ast.Raise) for item in sub.body + ), (ref, name, ast.unparse(sub.test)) + # The evidence-artifact tables. + for table in _EVIDENCE_ARTIFACT_TABLES: + for artifact, value in _artifact_table(table).items(): + if not artifact.endswith("_enum_domain"): + assert not _named(table[0], value, _HELPER_SCOPE), (table, artifact) + # The batteries and their phases. + batteries = _gate_batteries() + assert { + ref: name + for name, battery in batteries.items() + for ref in battery.phases.values() + } == _GATE_BATTERY_KERNELS + assert dict(batteries["spine"].phases) == { + "assembled": "uk.spine-gates@1", + "transferred": "uk.spine-gates@1", + } + assert dict(batteries["full"].phases) == { + "preflight": "uk.full-gates.preflight@1", + "terminal": "uk.full-gates@1", + } + + +def test_the_national_gates_key_on_the_gate_registrys_whole_reach() -> None: + """The national gates evaluate whichever gates their node's gate scope + names, so they key on everything the gate registry's bindings compute + through (microcosm#931 ruling 6): battery_bindings lists exactly that, + and each such kernel's hash covers it.""" + from microcosm.build.uk_runtime import battery_bindings + + reach = set(_helper_walk(_GATE_REGISTRY).reach) + listed = set(battery_bindings.UK_GATE_REGISTRY_MODULES) + assert listed == reach, sorted(listed ^ reach) + assert len(listed) == len(battery_bindings.UK_GATE_REGISTRY_MODULES) + assert { + module.__name__ for module in battery_bindings.uk_gate_registry_modules() + } == listed + kernels = _uk_graph_kernels() + for ref in _WHOLE_REGISTRY_KERNELS: + assert reach <= _hashed_modules(kernels[ref]), ref + + def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: """Each listed kernel stop is met only where its reason says (microcosm#931).""" import ast from microcosm.build.uk_runtime import ( - battery_bindings, + graph_evidence, graph_kernels, + graph_national, + graph_terminal, release_certification, spi_spine, ) + from microcosm.build.uk_runtime.graph_evidence import add_uk_spine_gate_nodes kernels = _uk_graph_kernels() namers: dict[tuple[str, str], dict[str, frozenset]] = {} - for ref, kernel in kernels.items(): - if ref in _HELD_KERNELS: - continue - for stop, by in _kernel_walk(type(kernel)).met.items(): - namers.setdefault(stop, {})[ref] = by + for ref, walks in _kernel_rule_walks().items(): + for keyed in walks.values(): + for stop, by in keyed.walk.met.items(): + held = namers.setdefault(stop, {}) + held[ref] = held.get(ref, frozenset()) | by # The fixture resolver resolves the stage's own transform: of what holds a # value, it names only the builder the stage rule starts from. @@ -2804,6 +3477,16 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: reach = _helper_walk(reason) hashed = _hashed_modules(kernels[f"uk.stage.{stage}@1"]) assert set(reach.reach) | {module for module, _ in reach.stops} <= hashed + # The spine gates rebuild their frame through _minimal_frame too, and no + # spine gate node names a stage, so none takes that branch. + assert "uk.spine-gates@1" in namers[reason] + gated = add_uk_spine_gate_nodes( + uk_spine_graph(), + spec=load_country_spec("uk"), + engine_identity="declaration-only", + ) + gate_nodes = [node for node in gated.nodes if node.kernel == "uk.spine-gates@1"] + assert gate_nodes and not any("stage" in node.params for node in gate_nodes) # The source codecs: only the create kernel's run loads a source through # them, and only inside its branch for an unbound root transform. The @@ -2841,21 +3524,33 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: ) # The gate registry: the bound spine derives its gate digests through it, - # in _manifest_digests, which returns the battery run's digests alone; - # the national battery evaluates through it and is held there. - registry = (battery_bindings.__name__, "UK_GATE_REGISTRY") - held_there = {ref for ref, stop in _HELD_KERNEL_STOPS if stop == registry} - assert held_there == {"uk.full.national-gates@1"} - assert { - ref: by for ref, by in namers[registry].items() if ref not in held_there - } == { + # in _manifest_digests, which returns the battery run's digests alone. + # The gate batteries' runs evaluate through it, and each phase is held + # past it to the gates it declares; the national gates' run evaluates + # through it too, and they are walked past it to its whole reach. + registry = _GATE_REGISTRY + assert namers[registry] == { "uk.full.bound_spine@1": frozenset( {(release_certification.__name__, "_manifest_digests")} ), + "uk.spine-gates@1": frozenset( + {(graph_evidence.__name__, "UKSpineGateKernel.run")} + ), + "uk.full-gates.preflight@1": frozenset( + {(graph_terminal.__name__, "UKFullGateKernel.run")} + ), + "uk.full-gates@1": frozenset( + {(graph_terminal.__name__, "UKFullGateKernel.run")} + ), } - assert namers[registry]["uk.full.national-gates@1"] == frozenset( - {("microcosm.build.uk_runtime.graph_national", "UKNationalGateKernel.run")} + assert set(namers[registry]) - {"uk.full.bound_spine@1"} == set( + _GATE_BATTERY_KERNELS ) + for ref in _WHOLE_REGISTRY_KERNELS: + assert registry not in _kernel_rule_walks()[ref][ref].walk.met, ref + assert _kernel_walk(type(kernels["uk.full.national-gates@1"])).met[ + registry + ] == frozenset({(graph_national.__name__, "UKNationalGateKernel.run")}) digests = _module_definitions( release_certification.__name__, _HELPER_SCOPE ).definitions["_manifest_digests"] @@ -2869,27 +3564,16 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: def test_every_hold_still_guards_a_module_its_kernel_leaves_out() -> None: """A hold lasts only while it is needed (microcosm#931 review): each held - kernel's walk still meets a module its hash leaves out, and so does the - walk past each held stop, so a hold that guards nothing fails here and - leaves its table. Until the ruling, such a kernel can replay a stale - output after an edit to one of those modules.""" + kernel's walk still meets a module its hash leaves out, so a hold that + guards nothing fails here and leaves its table. Until the ruling, such a + kernel can replay a stale output after an edit to one of those + modules.""" kernels = _uk_graph_kernels() - assert {ref for ref, _ in _HELD_KERNEL_STOPS} <= set(kernels) - assert not {ref for ref, _ in _HELD_KERNEL_STOPS} & set(_HELD_KERNELS) unneeded = [] for ref in _HELD_KERNELS: kernel = kernels[ref] if not set(_kernel_walk(type(kernel)).reach) - _hashed_modules(kernel): unneeded.append(ref) - for ref, stop in _HELD_KERNEL_STOPS: - assert stop in _KERNEL_STOPS, (ref, stop) - kernel = kernels[ref] - past = _kernel_walk( - type(kernel), - halts={item: why for item, why in _KERNEL_STOPS.items() if item != stop}, - ) - if not set(past.reach) - _hashed_modules(kernel): - unneeded.append((ref, stop)) assert not unneeded, unneeded @@ -2899,15 +3583,13 @@ def test_the_core_definitions_the_kernels_name_are_listed() -> None: the UK graph kernels' own code names, outside the held kernels, are exactly ``_KERNEL_CORE_NAMES``, and the executor stores every node's owned cells through the dtype tokens' map, as that entry's reason says.""" - kernels = _uk_graph_kernels() - named: set[tuple[str, str]] = set() - for ref, kernel in kernels.items(): - if ref not in _HELD_KERNELS: - named |= { - item - for item in _kernel_walk(type(kernel)).core_names - if not item[0].startswith("microcosm.frame.") - } + named = { + item + for walks in _kernel_rule_walks().values() + for keyed in walks.values() + for item in keyed.walk.core_names + if not item[0].startswith("microcosm.frame.") + } assert named == set(_KERNEL_CORE_NAMES), sorted(named ^ set(_KERNEL_CORE_NAMES)) patching = _helper_walk(("microcosm.graph.population", "patch"), follow_core=True) assert "dtype_for_token" in patching.reach["microcosm.graph.population"] @@ -3019,10 +3701,12 @@ def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> No """Control: the kernel rule follows what a kernel's methods name, through its bases by method resolution order (a base method it inherits, and one it overrides only where it calls ``super()``), and leaves out its - implementation hash, a listed refusal-only method, a base method it - overrides without ``super()`` and what lies behind a listed kernel stop, - which it reports with what named it. The recorder sees each module the - hash passes to ``source_hash``, in whichever module calls it.""" + implementation hash, a method only the hash calls, a listed refusal-only + method, a base method it overrides without ``super()`` and what lies + behind a listed kernel stop, which it reports with what named it. A + method the hash and the run both call stays. The recorder sees each + module the hash passes to ``source_hash``, in whichever module calls + it.""" import importlib import uuid @@ -3039,6 +3723,8 @@ def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> No "refused": "def check(x):\n return x < 0\n", "keyonly": "def salt():\n return 'salt'\n", "behind": "def beyond(x):\n return x * 5\n", + "resourced": "def fingerprint(x):\n return x * 7\n", + "shared": "def both(x):\n return x * 11\n", "table": "from .behind import beyond\n\nTABLE = {'a': beyond}\n", "keyed": ( "from microcosm.graph import source_hash\n\n" @@ -3063,10 +3749,16 @@ def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> No "from .keyed import key\n" "from .keyonly import salt\n" "from .refused import check\n" + "from .resourced import fingerprint\n" + "from .shared import both\n" "from .table import TABLE\n\n" "class Kernel(Base):\n" " def implementation_hash(self):\n" - " return salt() + key()\n\n" + " return salt() + key() + self._resources(1) + self._both(1)\n\n" + " def _resources(self, x):\n" + " return str(fingerprint(x))\n\n" + " def _both(self, x):\n" + " return str(both(x))\n\n" " def _admit(self, x):\n" " if check(x):\n" " raise ValueError('refused')\n\n" @@ -3074,6 +3766,7 @@ def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> No " return x\n\n" " def run(self, x):\n" " self._admit(x)\n" + " self._both(x)\n" " return computed(x) + super().run(x) + TABLE['a'](x)\n" ), } @@ -3094,6 +3787,7 @@ def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> No "computed": {"computed"}, "inherited": {"base_only"}, "assisted": {"kept"}, + "shared": {"both"}, } assert walk.met == { (f"{prefix}table", "TABLE"): frozenset({(f"{prefix}kernel", "Kernel.run")}) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py index 659f4705ac..224cba22dc 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py @@ -337,7 +337,9 @@ def test_released_stage_results_leave_every_stored_byte_and_no_result(tmp_path): } -def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions(): +def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions( + monkeypatch, +): country = load_country_spec("uk") spine = uk_spine_graph(country) endpoint = uk_spine_endpoint(spine) @@ -368,14 +370,29 @@ def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions(): assert "spine.gates.assembled" in compiled.predecessors[admitted] assert graph.node(admitted).params["spine_gate_phase"] == "assembled" assert "spine_gate_phase" not in graph.node("was_wealth").params - # Only the transferred phase evaluates the energy rake, so it alone keys - # on the module that sets the level; the assembled gate, which every later - # spine stage waits on, does not (the re-review of microcosm#1420, E). - from microcosm.build.uk_runtime import energy_pricing - from microcosm.graph.kernel import source_hash - - assert transferred.params["energy_pricing_code"] == source_hash(energy_pricing) - assert "energy_pricing_code" not in assembled.params + # Each phase keys on its own code: only the transferred phase evaluates + # the energy rake, so it alone keys on the module that sets the level, and + # the assembled gate, which every later spine stage waits on, does not + # (the re-review of microcosm#1420, E; microcosm#931 ruling 6). + from microcosm.build.uk_runtime import energy_pricing, graph_evidence + + codes = graph_evidence.UK_SPINE_GATE_PHASE_CODE + assert transferred.params["transferred_code"] == codes["transferred"]() + assert assembled.params["assembled_code"] == codes["assembled"]() + assert "transferred_code" not in assembled.params + assert "assembled_code" not in transferred.params + hashed = [] + with monkeypatch.context() as patch: + patch.setattr( + graph_evidence, + "source_hash", + lambda *objects: hashed.append(objects) or "", + ) + graph_evidence.uk_spine_transferred_only_code_hash.__wrapped__() + graph_evidence.uk_spine_assembled_only_code_hash.__wrapped__() + transferred_modules, assembled_modules = hashed + assert energy_pricing in transferred_modules + assert energy_pricing not in assembled_modules # The blocking posture travels with the gate and the admission it guards. assert assembled.params["synthetic_smoke"] is False assert graph.node(admitted).params["spine_gate_synthetic_smoke"] is False @@ -560,7 +577,7 @@ def test_sample_admission_holds_under_the_default_atomic_law(synthetic_smoke): def test_the_spine_gate_kernel_keys_on_its_checks_not_the_energy_level(monkeypatch): # The re-review of microcosm#1420, E: one kernel hash serves both phases, # so it covers the modules both read and not energy_pricing, which only - # the transferred phase's node param carries. + # the transferred phase's node param carries (microcosm#931 ruling 6). from microcosm.build import stochastic_assignment from microcosm.build.uk_runtime import ( battery_bindings, @@ -584,6 +601,7 @@ def recording(*objects): (modules,) = seen assert {stage_health, battery_bindings, graph_evidence.gate_battery} <= set(modules) # The latent-attribute checks' binomial band lives in stochastic_assignment - # (microcosm#931), so the gate keys on it too. - assert stochastic_assignment in modules + # (microcosm#931), and only the transferred phase declares those checks, + # so its own code keys on it, not the hash both phases share. + assert stochastic_assignment not in modules assert energy_pricing not in modules diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index 54da8fc6d1..13a7ed967e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -402,6 +402,7 @@ def test_full_preflight_persists_real_source_failures_without_matrix_diagnostics from microcosm.build.uk_runtime.graph_terminal import ( UKFullPreflightGateKernel, decode_full_gate_report, + uk_preflight_only_code_hash, ) from microcosm.graph.canonical import canonical_json @@ -430,6 +431,7 @@ def artifact(payload): "review_date": "2026-09-10", "sample_fraction": 1.0, "release_candidate": True, + "preflight_code": uk_preflight_only_code_hash(), }, artifacts={ "selection": artifact({"receipt": selection}), @@ -1464,9 +1466,11 @@ def test_the_gate_kernel_keys_the_solve_without_the_high_value_homes_code( monkeypatch, ): # The dense solve keys on the source preflight, which runs this kernel, so - # the report-only high-value homes code keys the calibrated node through - # its parameters and stays out of the kernel's hash (microcosm#1306 - # review). + # the report-only high-value homes evidence code keys the calibrated node + # through its parameters and stays out of the kernel's hash (microcosm#1306 + # review). high_value_homes' metric names, which the preflight's target + # surface reads, are in the kernel's hash, and the target kernels hash it + # too (microcosm#931 ruling 6). from microcosm.build.uk_runtime import ( graph_terminal, high_value_home_gates, @@ -1481,7 +1485,6 @@ def recording(*objects, **kwargs): return real(*objects, **kwargs) monkeypatch.setattr(graph_terminal, "source_hash", recording) - high_value = {high_value_home_gates.__name__, high_value_homes.__name__} # The preflight kernel inherits the calibrated kernel's code hash. for kernel in ( graph_terminal.UKFullGateKernel, @@ -1490,12 +1493,14 @@ def recording(*objects, **kwargs): kernel( coverage_engine=object(), engine_identity="fixture" ).implementation_hash() - assert not hashed[-1] & high_value, kernel.__name__ - # The hash is memoised per process; its undecorated body hashes the two - # high-value homes modules (and the diagnostics publication code, next - # test). + assert high_value_home_gates.__name__ not in hashed[-1], kernel.__name__ + assert high_value_homes.__name__ in hashed[-1], kernel.__name__ + # The hash is memoised per process; its undecorated body hashes the + # high-value homes evidence (and the diagnostics publication code, next + # test), and not what the kernel's hash holds. graph_terminal.uk_terminal_only_code_hash.__wrapped__() - assert high_value <= hashed[-1] + assert high_value_home_gates.__name__ in hashed[-1] + assert high_value_homes.__name__ not in hashed[-1] first = graph_terminal.uk_terminal_only_code_hash() calls = len(hashed) assert graph_terminal.uk_terminal_only_code_hash() == first @@ -1511,20 +1516,18 @@ def test_the_calibrated_node_keys_on_the_schema_9_publication_code_not_the_solve # dumped from CalibrationDiagnosticsV9. An edit to any of those modules # changes the output's bytes, so it must re-key the calibrated node (through # its terminal_code parameter); otherwise a store run with resume="auto" - # serves the old labels. Like the high-value homes code, it stays out of - # the two kernels' shared code hash. An edit to the catalogs, the schema - # or the label choices therefore never re-keys the preflight node or the - # solve; an edit to geography_constants still does, because the target - # kernels the preflight reads hash it too (microcosm#1492). + # serves the old labels. Like the high-value homes evidence code, it stays + # out of the two kernels' shared code hash. An edit to the catalogs, the + # schema or the label choices therefore never re-keys the preflight node + # or the solve. geography_constants is in the shared hash, since the + # preflight's ledger parity computes through it, and the target kernels + # the preflight reads hash it too (microcosm#1492; microcosm#931 ruling + # 6), so an edit to it re-keys both. import microcosm.calibrate.geography_constants as geography_constants_module import microcosm.calibrate.hierarchy_labels as hierarchy_labels_module import microcosm.diagnostics.catalogs as catalogs_module import microcosm.diagnostics.schema as diagnostics_schema_module - from microcosm.build.uk_runtime import ( - graph_terminal, - high_value_home_gates, - high_value_homes, - ) + from microcosm.build.uk_runtime import graph_terminal, high_value_home_gates hashed_module_names: list[set[str]] = [] real_source_hash = graph_terminal.source_hash @@ -1538,7 +1541,6 @@ def record_hashed_modules(*objects, **dependencies): catalogs_module.__name__, diagnostics_schema_module.__name__, hierarchy_labels_module.__name__, - geography_constants_module.__name__, } for kernel in ( graph_terminal.UKFullGateKernel, @@ -1548,29 +1550,32 @@ def record_hashed_modules(*objects, **dependencies): coverage_engine=object(), engine_identity="fixture" ).implementation_hash() assert not hashed_module_names[-1] & publication, kernel.__name__ + assert geography_constants_module.__name__ in hashed_module_names[-1] graph_terminal.uk_terminal_only_code_hash.__wrapped__() - assert hashed_module_names[-1] == { - high_value_home_gates.__name__, - high_value_homes.__name__, - *publication, - } + assert {high_value_home_gates.__name__, *publication} <= hashed_module_names[-1] + assert geography_constants_module.__name__ not in hashed_module_names[-1] -def test_the_calibrated_gates_refuse_a_stale_terminal_code() -> None: +@pytest.mark.parametrize("phase", ["preflight", "terminal"]) +@pytest.mark.parametrize("code", ["0" * 64, None]) +def test_each_gate_phase_refuses_a_stale_or_missing_phase_code(phase, code) -> None: + """Each full gate phase's node carries its own code, which the kernel + checks before it evaluates anything (microcosm#931 ruling 6).""" from types import SimpleNamespace - from microcosm.build.uk_runtime.graph_terminal import UKFullGateKernel + from microcosm.build.uk_runtime.graph_terminal import ( + UKFullGateKernel, + UKFullPreflightGateKernel, + ) - kernel = UKFullGateKernel(coverage_engine=object(), engine_identity="fixture") - context = SimpleNamespace( - params={ - "engine_identity": "fixture", - "phase": "terminal", - "terminal_code": "0" * 64, - }, - artifacts={}, + kernel = (UKFullPreflightGateKernel if phase == "preflight" else UKFullGateKernel)( + coverage_engine=object(), engine_identity="fixture" ) - with pytest.raises(ValueError, match="terminal-only code"): + params = {"engine_identity": "fixture", "phase": phase} + if code is not None: + params[f"{phase}_code"] = code + context = SimpleNamespace(params=params, artifacts={}) + with pytest.raises(ValueError, match=f"{phase}-only code"): kernel.run(context) @@ -1596,3 +1601,19 @@ def test_the_dense_gates_refuse_references_of_two_builds() -> None: _require_one_reference( {"input_mass_reference": national, "reference_surface": national} ) + + +def test_each_full_gate_node_carries_its_own_phase_code() -> None: + """The preflight and calibrated gate nodes each carry their phase's own + code, and only their own (microcosm#931 ruling 6).""" + from microcosm.build.uk_runtime.graph_terminal import UK_FULL_GATE_PHASE_CODE + + graph = _gate_nodes() + for node, phase in ( + ("uk.full.gates.preflight", "preflight"), + ("uk.full.gates.calibrated", "terminal"), + ): + params = graph.node(node).params + assert params["phase"] == phase + assert {key for key in params if key.endswith("_code")} == {f"{phase}_code"} + assert params[f"{phase}_code"] == UK_FULL_GATE_PHASE_CODE[phase]() From c7874fcec8c1e0a70077481483b3a053f968d057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:43:56 +0000 Subject: [PATCH 038/108] Hold the UK target and problem kernels to the derived binding rule (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María ruled on 9 Nov (ruling 6) that the derived binding rule replaces microcosm#1497's per-kernel import record for the UK target and problem kernels, that 0a9d20833's edit to that test's hash inputs stays, and that the false reason in transport/compile_modules.py is corrected with it. This commit does all three. It moves keys only. - The six kernels leave _HELD_KERNELS. The table, its reason and the hold's liveness test go, so no kernel stays held. test_uk_graph's kernel rule now walks the full build's target compilation, selection, measures and problem, and the national targets and problem, as it walks every other kernel. - Bindings. What all four full kernels compute through goes in the hash they share. What only one computes through goes in its own implementation hash, as for the size search and refit. - The shared hash gains the canonical encoder. - The compilation adds chronicle_epoch, whose schema versions and fact-key epochs ledger_artifact checks and records, and frs_relationships, whose ONS household type domain ledger_targets checks the bindings against. - The measures add the 15 modules that resolving the measures and compiling their national problem compute through: graph_population, national_frame, atomic_area_support, frame.materialize, the engine adapter, dfe_spring_term, frs_annual_income, hmrc_income, income_tax_rate_classes, lcfs_consumption, uc_target_measurements, chronicle_epoch, national_calibration, local_rowwise and calibrate.monetary_binding. - The problem adds graph_population, national_frame, local_geography and calibrate.monetary_binding. - The national kernels' shared hash gains chronicle_epoch, graph_targets and the canonical encoder. The national targets add frs_relationships. The national problem adds the measures' modules except chronicle_epoch and national_calibration, which its shared hash binds, plus diagnostics and gate_battery. - The measures and the national problem also bind frs_spine for one name, the annuity and trust column. The resolver reads zeros where a frame lacks it, so on a bound spine a renamed constant would change the problem without re-keying it. - The clone index columns' name stays listed. Its reason now covers the dense measures, whose population is the expand's and which read it only to split into engine blocks, refusing a frame without it. It also covers the national problem, which resolves in one block and never reads it. The kernel rule's test checks these facts. - No kernel loses a module, though some hash modules the walk does not reach: the selection hashes 51 and computes through 6. - test_uk_target_compile_modules. UNCLASSIFIED_IMPORTS, the kernel groups and the record check go, and the docstring says the derived rule now holds these kernels. The closure check of the shared list, the hash tests and HASH_INPUTS stay, with 0a9d20833's seven entries. Three of them (calibrate.monetary_binding, the engine adapter and frame.materialize) are now also imported in the national problem's hash, and their reason says so. A new test checks what the derived rule does not: - each hash input is still imported where listed; - a kernel hashes one only where another module it hashes imports it. So no kernel hashes the compile list or the solve fingerprint's module. - target_compile_modules.py's docstring no longer names the retired record. The file is a hash input, so the edit re-keys nothing. - transport/compile_modules.py. _GRAPH_RUNTIME said no kernel in the repository hashes a microcosm.graph module, which was false. It now says: - what the listed modules use those modules for; - that the compile kernels hash none of them; - that other kernels hash the canonical encoder, each UK graph kernel whose code computes through it among them. No kernel hashes compile_modules.py, so the edit re-keys nothing. - The changelog fragment moves the six kernels from held to bound. What moves: node keys only. No value, declaration or generated file moves. - 11 of the 99 UK kernel hashes move: the six, and five that hash graph_targets.py or graph_national.py (the preflight and calibrated gates, the rotated holdout, the national gates and the readback). - Full build with its gates, in the country adapter's in-graph composition: 14 of 113 nodes re-key. They are the target compilation, selection, measures and problem, the pool checkpoint, the preflight, the dense solve, the size search, draw and refit, the selected and calibrated populations, the calibrated gates and the holdout. Every node after them re-keys by chaining. The spine and the dense population and geography nodes keep their keys. - National line: 5 of 6 nodes re-key (the targets, the problem, the dense solve, the calibrated population and the national gates). The bound spine checkpoint keeps its key. - The gated spine (81 nodes) and the transport graphs keep their keys. source_stages.json, the coverage manifest, uk_spine.json and the H2 fixture do not move. Tests: all synthetic or code facts. - test_uk_graph and test_uk_target_compile_modules pass (104 tests), as do 14 related engine-free files (365 tests). Through the engine slot, engine/uk/test_uk_full_target_graph passes (8 tests). - 19 mutations on a scratch copy are caught: - each of the six kernels, or a helper the measures compute through, newly computing through an unhashed module, or reading a lazy export; - a module leaving each kind of hash: the shared, the compilation's, the measures', the problem's, the national shared and the national problem's, frs_spine among them; - a kernel hashing a hash input; - the national problem resolving in blocks, and the build moving the measures' population. A twentieth, naming the module by a computed string, passes, as the rule documents; its static form is caught. - Ruff is clean on the six edited Python files. source_stages.json and the coverage manifest are current. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-helper-bindings.changed.md | 17 +- .../build/transport/compile_modules.py | 9 +- .../build/uk_runtime/graph_national.py | 87 ++- .../build/uk_runtime/graph_targets.py | 97 ++- .../uk_runtime/target_compile_modules.py | 13 +- .../tests/engine_free/uk/test_uk_graph.py | 137 ++-- .../uk/test_uk_target_compile_modules.py | 617 +++--------------- 7 files changed, 357 insertions(+), 620 deletions(-) diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index cb4f66054d..9d7ac4c1f5 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,4 +1,4 @@ -Every UK spine stage now hashes each first-party module its transform computes through, and each stage that calls the engine also each one the injected engine computes through for it. Every other UK graph kernel, except six held for a ruling, hashes each one its own code computes through, and each phase of a gate battery each one its own gates compute through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. +Every UK spine stage now hashes each first-party module its transform computes through, and each stage that calls the engine also each one the injected engine computes through for it. Every other UK graph kernel hashes each one its own code computes through, the target and problem kernels among them, and each phase of a gate battery each one its own gates compute through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. - **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage, and for a stage that calls the engine also at the engine the builds inject (see The engine), and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports, module attributes and `type` aliases, which read as assignments of their values. So do the names `microcosm.graph` exports lazily through its module `__getattr__`: the walk resolves each to the runtime module that defines it, from the table that `__getattr__` reads, and a test checks that every first-party module with a module `__getattr__` is listed. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: - six listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels); @@ -35,7 +35,7 @@ Every UK spine stage now hashes each first-party module its transform computes t The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input`, `gates` and `spi_support`, whose replacement strata the sample reads. - **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. Through `graph_implementation_dependencies`, which the root node hashes, it declares `national_sampling` at every rung, since it resets the root's indexes at f100 too, and `frame_sampling` only below f100, where it samples. Its behaviour is unchanged. -- **Kernels.** A second test applies the rule to every kernel the UK registries register but the six held below: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; the national gates and readback; and, phase by phase, the spine gates and the full build's preflight and calibrated gates (see Gate batteries). It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key, or a method only that hash calls, such as the full gates' `_country_resources`. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at four listed kernel stops, each with a reason a test checks: +- **Kernels.** A second test applies the rule to every kernel the UK registries register: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the full build's target compilation, selection, measures and problem and the national targets and problem (see Target and problem kernels); the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; the national gates and readback; and, phase by phase, the spine gates and the full build's preflight and calibrated gates (see Gate batteries). It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key, or a method only that hash calls, such as the full gates' `_country_resources`. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at four listed kernel stops, each with a reason a test checks: - the hermetic fixture's transform resolver, since the stage rule holds each stage's transform and, through its second root, the engine the resolver's builder injects into the engine stages (see The engine); - `_minimal_frame`'s rebuild of the SPI support allocation's mass record, which only `hmrc_spi_income_spine` takes, and whose reach that stage's node hashes. The spine gates rebuild their frame through `_minimal_frame` too, and no spine gate node names a stage (checked); - `UK_GATE_REGISTRY`, which the bound spine reaches only for its checkpoint's gate digests. The gate batteries and the national gates evaluate through it, and the rule holds them past it (see Gate batteries); @@ -58,11 +58,16 @@ Every UK spine stage now hashes each first-party module its transform computes t - the national gates: `gates`, microcosm.calibrate's `artifacts`, `matrix`, `registry`, `target`, `hierarchy`, `monetary_binding` and `solve`, `graph_targets`, `graph_population`, `national_frame`, `etb_services`, `cgt_projection`, `donor_uprating` and the canonical encoder, and everything the gate registry's bindings compute through (see Gate batteries); - the national readback: `artifact_files`, `trace`, `frame.materialize`, the engine adapter, `geography_ladder`, `graph_population`, `hmrc_uprating` and the canonical encoder. - The LAD23 names' pin stays listed, now for every kernel that reads it. The geographic clone index columns' name (`rowwise_dataset.ladder_clone_index_column`) is listed for the calibrated and national gates, whose support-family fold reads it: only the dense expand writes those columns, and it hashes `rowwise_dataset`; the national frame has no ladder, and the fold keys on the column only where the table carries it. `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside the national gates' and readback's hashes. + The LAD23 names' pin stays listed, now for every kernel that reads it. The geographic clone index columns' name (`rowwise_dataset.ladder_clone_index_column`) is listed for the calibrated and national gates, whose support-family fold reads it: only the dense expand writes those columns, and it hashes `rowwise_dataset`; the national frame has no ladder, and the fold keys on the column only where the table carries it. It is listed for the dense measures and the national problem too (see Target and problem kernels). `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside implementation hashes: the national gates' and readback's, and for three of them the national problem's. - **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Another test checks each part of that model against the code. So an edit only the transferred phase computes through re-keys neither the assembled gate nor the spine stages that wait on it, and one only the terminal phase computes through re-keys neither the preflight nor the dense solve keyed on it. - The spine gates' hash binds the 11 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants` and `calibrate.geography_constants`, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. As before, the spine gate kernel does not check its phase's code. - The full gates' hash binds the 26 modules both phases compute through. It loses `local_rowwise`, `weighted_integrity` and `diagnostics`, which only the terminal phase computes through. The preflight node carries a new `preflight_code`: the shared ledger's registry parity, behind the ledger compile parity gates only the preflight declares. The calibrated node's `terminal_code` now hashes 37 modules. `high_value_homes` and `calibrate.geography_constants` move into the shared hash, since the preflight computes through them too; the target kernels hash both, so an edit to either already re-solved. The kernel refuses a stale or missing `preflight_code`, as it refuses a stale `terminal_code`. - The national gates evaluate whichever gates their node's scope names, so their hash binds everything the gate registry's bindings compute through. That is 50 modules, which `battery_bindings.UK_GATE_REGISTRY_MODULES` lists and a test holds exactly to the registry's derived reach. Their stop at the registry, the one held kernel stop, goes. -- **Held for a ruling.** Six kernels stay outside the rule, each listed with its reason. Until the ruling, a held kernel can replay a stale output after an edit to a module it computes through but does not hash; a test checks that every held kernel's walk still meets such a module, so a hold cannot outlive its need: - - the target compilation, selection, measures and problem kernels, and the national targets and problem. microcosm#1497's import record holds their keys and may only shrink. Binding what the rule finds would add to it every unhashed import of each module bound. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. +- **Target and problem kernels.** As María ruled on 9 Nov, the derived rule replaces microcosm#1497's per-kernel import record for the full build's target compilation, selection, measures and problem and the national targets and problem, which that record held until now. No kernel stays held. + - `test_uk_target_compile_modules` loses the record (`UNCLASSIFIED_IMPORTS`), its kernel groups and the check that read them. It keeps the shared list's closure check, the tests of what the kernels hash and its hash inputs, the seven above among them. A new check covers what the derived rule does not: each hash input is still imported where listed, and a kernel hashes one only where another module it hashes imports it too. So no kernel hashes the compile list or the solve fingerprint's module, and an edit to either re-keys none. + - What all four full kernels compute through goes in the hash they share, and what only one computes through goes in its own implementation hash, as for the size search and refit. The shared hash gains the canonical encoder. The compilation adds `chronicle_epoch`, whose schema versions and fact-key epochs `ledger_artifact` checks the facts against and records, and `frs_relationships`, whose ONS household type domain `ledger_targets` checks the household-type bindings against. The measures add the 15 modules that resolving the measures and compiling their national problem compute through: `graph_population` and `national_frame`, for the frame they rebuild; `atomic_area_support`, `frame.materialize` and the engine adapter, for the engine's scratch frame; `dfe_spring_term`, `frs_annual_income`, `hmrc_income`, `income_tax_rate_classes`, `lcfs_consumption` and `uc_target_measurements`, for the resolver's inputs and measures; `chronicle_epoch`, for the vendored facts' schema checks; `national_calibration`, for the measure inputs it injects; `local_rowwise`, for the national rows; and `calibrate.monetary_binding`, for the constraint matrix's check. The problem adds `graph_population`, `national_frame`, `local_geography` (the local surface matrix's area columns) and `calibrate.monetary_binding`. + - The national targets' and problem's shared hash gains `chronicle_epoch`, `graph_targets` (the register codec) and the canonical encoder. The national targets add `frs_relationships`. The national problem adds the measures' modules but `chronicle_epoch` and `national_calibration`, which its shared hash already binds, and adds `diagnostics` and `gate_battery`, whose target geography levels and JSON-safe values its payload records. + - The measures and the national problem also bind `frs_spine` for one name, the annuity and trust column. The resolver reads that column where the frame carries it and reads zeros where it does not, so on the national line's bound spine a renamed constant would change the problem without re-keying it. The clone index columns' name stays listed for them: the dense measures read it only to split their frame into the engine blocks they are asked for, and refuse a frame without it, and their population is the expand's; the national problem resolves its measures in one block, which never reads it. A test checks these facts. + - No kernel drops a module it hashed before, though each still hashes some modules its walk does not reach. The full kernels' shared hash still binds the whole compile list and the modules beside it in all four, so the selection's key covers 51 modules while its code computes through 6. + - `transport/compile_modules.py`'s reason for leaving the graph runtime off the transport compile list said that no kernel in the repository hashes a `microcosm.graph` module, which was untrue. It now says what the listed modules use those modules for, that the compile kernels hash none of them, and that other kernels hash the canonical encoder, each UK graph kernel whose code computes through it among them. No kernel hashes that file, so the edit re-keys nothing. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. The target and problem kernels' keys move once with their new bindings. `graph_targets.py` and `graph_national.py`, which define them, changed with them, and the preflight and calibrated gates, the rotated holdout and the national gates and readback each hash one or both of those files too. So in the full build the target compilation, selection, measures and problem, the pool checkpoint (`uk.full.pool`), the preflight, the rotated holdout, and the dense solve and every node after it re-key, and on the national line every node after its bound spine root; no spine node and no transport node re-keys with them. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py index e38932c372..58c0437ae5 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py @@ -51,9 +51,12 @@ "validators do not use" ) _GRAPH_RUNTIME = ( - "the graph runtime (kernel protocol, declarations and the canonical JSON " - "encoder node keys are computed with); no kernel in the repository " - "hashes a microcosm.graph module" + "the graph runtime the listed modules read a node's context, declare " + "artifact types and encode documents with: the kernel protocol, the " + "declarations and the canonical JSON encoder node keys are computed " + "with. The compile kernels hash none of the three; other kernels hash " + "the encoder, among them each UK graph kernel whose code computes " + "through it (microcosm#931)" ) EXCLUDED_DIRECT_IMPORTS: Mapping[str, str] = MappingProxyType( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 147b42bdf5..95af501c00 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -474,8 +474,9 @@ class _NationalKernel(KernelBase): def implementation_hash(self) -> str: import microcosm.calibrate.artifacts as calibrate_artifacts import microcosm.calibrate.matrix as calibrate_matrix + from microcosm.build import chronicle_epoch - from . import receipt_counts, weighted_integrity + from . import graph_targets, receipt_counts, weighted_integrity from .target_compile_modules import uk_target_compile_modules implementation = source_hash( @@ -508,6 +509,16 @@ def implementation_hash(self) -> str: weighted_integrity, receipt_counts, *uk_target_compile_modules(), + # What both kernels also compute through (microcosm#931): the + # Chronicle schema versions and fact-key epochs the facts are + # checked against (chronicle_epoch), the register codec + # (graph_targets) and the canonical encoding of their artifacts. + # What only one of them computes through, it hashes in its own + # implementation hash below; test_uk_graph derives each kernel's + # set from what its code names. + chronicle_epoch, + graph_targets, + canonical_json, ) return hashlib.sha256( canonical_json( @@ -522,6 +533,21 @@ def implementation_hash(self) -> str: class UKNationalTargetKernel(_NationalKernel): ref = "uk.full.national_targets@1" + def implementation_hash(self) -> str: + from . import frs_relationships + + # The national register is compiled through ledger_targets, which + # checks the household-type bindings against the ONS household type + # domain (frs_relationships) (microcosm#931). + return hashlib.sha256( + canonical_json( + { + "national": super().implementation_hash(), + "targets": source_hash(frs_relationships), + } + ) + ).hexdigest() + def run(self, context: KernelContext) -> KernelResult: inputs = load_uk_national_target_inputs( context.sources["uk_ledger_facts"], @@ -672,6 +698,65 @@ class UKNationalProblemKernel(_NationalKernel): def __init__(self, *, resolver_factory=UKMeasureResolver) -> None: self.resolver_factory = resolver_factory + def implementation_hash(self) -> str: + import microcosm.calibrate.monetary_binding as monetary_binding + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import gate_battery + from . import ( + atomic_area_support, + dfe_spring_term, + diagnostics, + frs_annual_income, + frs_spine, + graph_population, + hmrc_income, + income_tax_rate_classes, + lcfs_consumption, + local_rowwise, + national_frame, + uc_target_measurements, + ) + + # The problem also computes through what the measure resolver and the + # engine run on the bound frame (microcosm#931), as the dense line's + # measures do: the frame it rebuilds on national_frame's schema + # (graph_population, national_frame), the engine's scratch frame + # (atomic_area_support, frame_materialize and the engine adapter's + # contracts), the resolver's inputs and measures (dfe_spring_term, + # frs_annual_income, frs_spine's annuity and trust column, + # hmrc_income, income_tax_rate_classes, lcfs_consumption and + # uc_target_measurements), the national rows (local_rowwise), the + # constraint matrix's monetary check (monetary_binding), and the + # target geography levels (diagnostics) and JSON-safe values + # (gate_battery) its payload records. + return hashlib.sha256( + canonical_json( + { + "national": super().implementation_hash(), + "problem": source_hash( + atomic_area_support, + dfe_spring_term, + diagnostics, + frs_annual_income, + frs_spine, + graph_population, + hmrc_income, + income_tax_rate_classes, + lcfs_consumption, + local_rowwise, + national_frame, + uc_target_measurements, + gate_battery, + monetary_binding, + frame_materialize, + uk_engine_adapter, + ), + } + ) + ).hexdigest() + def run(self, context: KernelContext) -> KernelResult: frame = context_frame(context) registry_document = _registry_artifact(context) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index ff981b7994..3473d7c3e8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -259,6 +259,11 @@ def implementation_hash(self) -> str: weighted_integrity, receipt_counts, *uk_target_compile_modules(), + # The canonical encoding of the artifacts all four kernels store + # and read (microcosm#931). What only one of them computes + # through, it hashes in its own implementation hash below; + # test_uk_graph derives each kernel's set from what its code names. + canonical_json, ) # The solve fingerprint: every packaged resource but the gate-only # registers, which no compile module reads (microcosm#1301), so a @@ -276,6 +281,25 @@ def implementation_hash(self) -> str: class UKFullTargetCompilationKernel(_TargetKernel): ref = "uk.full.target_compilation@1" + def implementation_hash(self) -> str: + from microcosm.build import chronicle_epoch + + from . import frs_relationships + + # The compilation also computes through the Chronicle schema versions + # and fact-key epochs ledger_artifact checks the facts against and + # records in their provenance (chronicle_epoch), and the ONS household + # type domain ledger_targets checks the household-type bindings + # against (frs_relationships) (microcosm#931). + return hashlib.sha256( + canonical_json( + { + "targets": super().implementation_hash(), + "compilation": source_hash(chronicle_epoch, frs_relationships), + } + ) + ).hexdigest() + def run(self, context: KernelContext) -> KernelResult: from .full_targets import load_uk_full_target_inputs @@ -471,6 +495,63 @@ def run(self, context: KernelContext) -> KernelResult: class UKFullMeasureKernel(_TargetKernel): ref = "uk.full.measures@1" + def implementation_hash(self) -> str: + import microcosm.calibrate.monetary_binding as monetary_binding + from microcosm.build import chronicle_epoch + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from . import ( + atomic_area_support, + dfe_spring_term, + frs_annual_income, + frs_spine, + graph_population, + hmrc_income, + income_tax_rate_classes, + lcfs_consumption, + national_calibration, + national_frame, + uc_target_measurements, + ) + + # The measures also compute through what the measure resolver and the + # engine blocks run (microcosm#931): the frame they rebuild on + # national_frame's schema (graph_population, national_frame), the + # engine's scratch frame (atomic_area_support, frame_materialize and + # the engine adapter's contracts), the resolver's inputs and measures + # (dfe_spring_term, frs_annual_income, frs_spine's annuity and trust + # column, hmrc_income, income_tax_rate_classes, lcfs_consumption and + # uc_target_measurements), the vendored facts' schema checks + # (chronicle_epoch), the measure inputs the calibration frame adapter + # injects (national_calibration), the national rows (local_rowwise) + # and the constraint matrix's monetary check (monetary_binding). + return hashlib.sha256( + canonical_json( + { + "targets": super().implementation_hash(), + "measures": source_hash( + atomic_area_support, + chronicle_epoch, + dfe_spring_term, + frs_annual_income, + frs_spine, + graph_population, + hmrc_income, + income_tax_rate_classes, + lcfs_consumption, + local_rowwise, + national_calibration, + national_frame, + uc_target_measurements, + monetary_binding, + frame_materialize, + uk_engine_adapter, + ), + } + ) + ).hexdigest() + def run(self, context: KernelContext) -> KernelResult: gate = json.loads(context.artifacts["geography_gate"].payload) if not gate["passed"]: @@ -668,6 +749,10 @@ class UKFullProblemKernel(_TargetKernel): ref = "uk.full.problem@1" def implementation_hash(self) -> str: + import microcosm.calibrate.monetary_binding as monetary_binding + + from . import graph_population, local_geography, national_frame + # The loss weights the problem binds are built by # ``local_rowwise.prepare_uk_full_solve`` from the rules in # ``target_weights`` and ``national_doctrine`` (microcosm#1124), so an @@ -677,7 +762,11 @@ def implementation_hash(self) -> str: # ``calibrate.solve.default_target_loss_scales`` (microcosm#1492). All # of ``calibrate.solve`` is hashed: ``calibrate.adam@1``, whose hash # every UK solve kernel embeds, already hashes it, so an edit there - # re-solves anyway and re-keys only this node besides. + # re-solves anyway and re-keys only this node besides. The problem + # also computes through the frame it rebuilds on national_frame's + # schema (graph_population, national_frame), the area columns of the + # local surface matrix local_rowwise builds (local_geography) and the + # constraint matrix's monetary check (monetary_binding) (microcosm#931). return hashlib.sha256( canonical_json( { @@ -689,6 +778,12 @@ def implementation_hash(self) -> str: geography_constants, calibrate_solve, ), + "problem": source_hash( + graph_population, + national_frame, + local_geography, + monetary_binding, + ), } ) ).hexdigest() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py index b832f200fd..2706398a0b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/target_compile_modules.py @@ -20,11 +20,14 @@ ``microcosm.calibrate.target``, which the registry imports). The transport compile modules already hashed those three. -The exclusions cover only the listed modules' own imports. A second test -walks every module each target and problem kernel hashes, its own module and -the modules hashed beside this tuple included, and records the imports no -review has classified yet (microcosm#1497). This module is not hashed, so an -edit to a reason re-keys nothing; an edit to the tuple re-keys the kernels. +The exclusions cover only the listed modules' own imports, and keep a module +off the list, not out of a kernel's hash. Each target and problem kernel also +hashes, beside this tuple, every first-party module its own code computes +through, some of the excluded ones among them: the derived binding rule in +test_uk_graph holds each kernel's hash to that, in place of the per-kernel +import record microcosm#1497 kept (microcosm#931). This module is not hashed, +so an edit to a reason re-keys nothing; an edit to the tuple re-keys the +kernels. """ from __future__ import annotations diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 84120f3078..58400a597d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -1303,7 +1303,12 @@ def changed_module_bytes(path): "frame, so their key chains on the expand's; the national gates fold " "the national frame, which no ladder expands, and the fold keys on the " "column only where the table carries it " - "(diagnostics.uk_support_family_weights)" + "(diagnostics.uk_support_family_weights). The dense measures read it " + "only to split their frame into the engine blocks their engine_blocks " + "parameter asks for, and refuse a frame without it; their population " + "is the expand's, so their key chains on it too. The national problem " + "resolves its measures in one block, which never reads it " + "(full_measure._engine_block_frames; checked)" ), } #: Where the kernel rule stops, each with why a kernel that names the @@ -1412,30 +1417,6 @@ def changed_module_bytes(path): "that artifact is absent", ), } -#: Kernels the rule does not yet hold, each with why, all held for a ruling -#: (microcosm#931): the target kernels, whose keys microcosm#1497's import -#: record holds. Until the ruling, a held kernel can replay a stale output -#: after an edit to a module it computes through but does not hash; -#: test_every_hold_still_guards_a_module_its_kernel_leaves_out checks that -#: each still does, so a hold cannot outlive its need. -_TARGET_RECORD_REASON = ( - "test_uk_target_compile_modules holds this kernel's key to " - "microcosm#1497's import record, which lists each import of a module the " - "kernel hashes that it leaves unhashed and may only shrink. Binding what " - "this rule finds it computing through would add to the record every such " - "import of each module bound" -) -_HELD_KERNELS: dict[str, str] = { - ref: _TARGET_RECORD_REASON - for ref in ( - "uk.full.target_compilation@1", - "uk.full.target_selection@1", - "uk.full.measures@1", - "uk.full.problem@1", - "uk.full.national_targets@1", - "uk.full.national_problem@1", - ) -} #: Packages that export names lazily, through a module ``__getattr__``, each #: with the table, of name to the submodule defining it, that ``__getattr__`` #: reads. The walk resolves such a name to the module that defines it and @@ -3038,7 +3019,8 @@ def _kernel_rule_walks() -> MappingProxyType: battery's kernel has one per phase it runs, labelled ``" "`` (``_battery_phase_walk``), against its hash with the phase's own code. A class with several kernels (the spine's stage kernels) is walked once. - The held kernels (``_HELD_KERNELS``) are left out.""" + Every UK graph kernel is walked, the target and problem kernels among + them (microcosm#931 ruling 6).""" kernels = _uk_graph_kernels() batteries = _gate_batteries() @@ -3048,10 +3030,6 @@ class with several kernels (the spine's stage kernels) is walked once. rule = {} for kernel_class, members in by_class.items(): refs = {ref for ref, _ in members} - held = {ref in _HELD_KERNELS for ref in refs} - assert len(held) == 1, kernel_class - if held == {True}: - continue if refs & set(_GATE_BATTERY_KERNELS): assert refs <= set(_GATE_BATTERY_KERNELS), kernel_class for ref, kernel in members: @@ -3088,7 +3066,8 @@ class with several kernels (the spine's stage kernels) is walked once. def test_the_kernel_rule_sees_every_kernel_the_uk_runtime_defines() -> None: """The kernels the rule enumerates are every kernel class the UK runtime and the shared atomic geography define, so a new one cannot escape it, - and the held kernels are among them (microcosm#931).""" + and the gate batteries' and whole-registry kernels are among them, each + listed once (microcosm#931).""" import ast import importlib @@ -3119,11 +3098,7 @@ def test_the_kernel_rule_sees_every_kernel_the_uk_runtime_defines() -> None: defined.add(getattr(importlib.import_module(module), node.name)) kernels = _uk_graph_kernels() assert {type(kernel) for kernel in kernels.values()} == defined - listed = ( - set(_HELD_KERNELS), - set(_GATE_BATTERY_KERNELS), - set(_WHOLE_REGISTRY_KERNELS), - ) + listed = (set(_GATE_BATTERY_KERNELS), set(_WHOLE_REGISTRY_KERNELS)) for table in listed: assert table <= set(kernels) assert sum(map(len, listed)) == len(set().union(*listed)) @@ -3145,14 +3120,25 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: gate battery is held to it one phase at a time, each phase's key being the battery's hash with the phase's own code (``_battery_phase_walk``), and the national gates past the gate registry's stop - (``_WHOLE_REGISTRY_KERNELS``; microcosm#931 ruling 6). The held kernels - (``_HELD_KERNELS``) are not held to it. A name microcosm.graph exports - lazily, through its module ``__getattr__``, is followed to the runtime - module that defines it like any other (``_LAZY_EXPORT_TABLES``): the - kernels' capability and context records are listed core names, and the - create kernel's ``load_source``, which it calls only where no root - transform is bound, is a listed stop.""" - from microcosm.build.uk_runtime import graph_population, rowwise_geography + (``_WHOLE_REGISTRY_KERNELS``; microcosm#931 ruling 6). The target and + problem kernels are held to it as every other kernel is: it replaces + microcosm#1497's per-kernel import record for them (microcosm#931 ruling + 6), and test_uk_target_compile_modules keeps only what it does not cover. + A name microcosm.graph exports lazily, through its module ``__getattr__``, + is followed to the runtime module that defines it like any other + (``_LAZY_EXPORT_TABLES``): the kernels' capability and context records are + listed core names, and the create kernel's ``load_source``, which it calls + only where no root transform is bound, is a listed stop.""" + import ast + import inspect + + from microcosm.build.uk_runtime import ( + graph_build, + graph_national, + graph_population, + graph_targets, + rowwise_geography, + ) kernels = _uk_graph_kernels() unhashed: dict[str, list[str]] = {} @@ -3170,6 +3156,48 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: unlisted ^ set(_KERNEL_UNBOUND_NAMES) ) assert met == set(_KERNEL_STOPS), sorted(met ^ set(_KERNEL_STOPS)) + # The clone index columns' reason for the measures. The dense measures' + # population is the expand's: append_uk_target_nodes defaults it, and the + # full build leaves the default. The national problem resolves its + # measures in one block, and the engine blocks return a one-block frame + # before they read the columns' name. + population = inspect.signature(graph_targets.append_uk_target_nodes).parameters + assert population["population"].default == "uk.full.expand" + + def calls(module, name: str) -> list: + tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) + return [ + sub + for sub in ast.walk(tree) + if isinstance(sub, ast.Call) + and getattr(sub.func, "id", getattr(sub.func, "attr", None)) == name + ] + + appending = calls(graph_build, "append_uk_target_nodes") + assert appending and all( + "population" not in {keyword.arg for keyword in call.keywords} + for call in appending + ) + resolving = calls(graph_national, "resolve_uk_full_measures") + assert resolving and all( + [ast.unparse(item.value) for item in call.keywords if item.arg == "blocks"] + == ["1"] + for call in resolving + ) + blocks = _module_definitions( + "microcosm.build.uk_runtime.full_measure", _HELPER_SCOPE + ).definitions["_engine_block_frames"] + statements = [ast.unparse(statement) for statement in blocks.body] + (one,) = [ + index + for index, statement in enumerate(blocks.body) + if isinstance(statement, ast.If) + and ast.unparse(statement.test) == "blocks == 1" + and isinstance(statement.body[0], ast.Return) + ] + assert all( + "ladder_clone_index_column" not in statement for statement in statements[:one] + ) # The dense population kernels that share _PopulationKernel's hash # (sample, normalize, expand, locations, geography mapping, pool and # gate) mint their clones through rowwise_geography's helpers. @@ -3562,27 +3590,12 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: } -def test_every_hold_still_guards_a_module_its_kernel_leaves_out() -> None: - """A hold lasts only while it is needed (microcosm#931 review): each held - kernel's walk still meets a module its hash leaves out, so a hold that - guards nothing fails here and leaves its table. Until the ruling, such a - kernel can replay a stale output after an edit to one of those - modules.""" - kernels = _uk_graph_kernels() - unneeded = [] - for ref in _HELD_KERNELS: - kernel = kernels[ref] - if not set(_kernel_walk(type(kernel)).reach) - _hashed_modules(kernel): - unneeded.append(ref) - assert not unneeded, unneeded - - def test_the_core_definitions_the_kernels_name_are_listed() -> None: """The kernel version of the stage rule's core check (microcosm#931 review): the definitions of the core modules other than the Frame's that - the UK graph kernels' own code names, outside the held kernels, are - exactly ``_KERNEL_CORE_NAMES``, and the executor stores every node's - owned cells through the dtype tokens' map, as that entry's reason says.""" + the UK graph kernels' own code names are exactly ``_KERNEL_CORE_NAMES``, + and the executor stores every node's owned cells through the dtype + tokens' map, as that entry's reason says.""" named = { item for walks in _kernel_rule_walks().values() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index ece2d8dc3b..b02a6a6e68 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -1,14 +1,17 @@ """Both UK target kernels hash every module that decides a target value. -Two checks pin the closure, as for the transport compile kernels: - -- the shared list's own: every first-party import of a module in - ``UK_TARGET_COMPILE_MODULE_NAMES`` is listed, or excluded with a reason; -- each kernel's: the walk from every module a target or problem kernel - hashes, its own module included, meets only modules the kernel hashes, the - list's exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS`` for - that kernel's group, which no review has classified yet (microcosm#1497). - That mapping may only shrink. +Two checks pin the closure: + +- the shared list's own, as for the transport compile kernels: every + first-party import of a module in ``UK_TARGET_COMPILE_MODULE_NAMES`` is + listed, or excluded with a reason; +- each kernel's: the derived binding rule in test_uk_graph.py holds what each + target and problem kernel's hash covers to what its code computes through + (``test_every_uk_graph_kernel_hashes_each_helper_it_computes_through``). It + replaced microcosm#1497's per-kernel import record for these kernels + (microcosm#931 ruling 6), so this file keeps only what that rule does not + cover: the modules a kernel's file imports only to build its hash + (``HASH_INPUTS``) stay out of the hashes. """ from __future__ import annotations @@ -40,7 +43,6 @@ assert_an_edit_rekeys, first_party_imports, hashed_module_names, - recorded_import_mismatches, unlisted_imports, ) @@ -160,8 +162,9 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): ) #: Code the target kernels ran without hashing it, found by the review of -#: microcosm#1492, and the kernels that now hash it. All four full-graph -#: kernels share one hash, so what one of them runs re-keys all four. +#: microcosm#1492, and the kernels that now hash it. These modules are in the +#: hash all four full-graph kernels share, so what one of them runs re-keys +#: all four. _RUN_MODULES = { # uk.full.target_selection@1 calls select_targets. "microcosm.calibrate.target_selection": _FULL_GRAPH, @@ -179,30 +182,14 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "microcosm.build.uk_runtime.ledger_surface": _FULL_GRAPH + _NATIONAL_GRAPH, } -#: The kernels grouped by shared hash: the full graph's target, selection and -#: measures kernels share _TargetKernel's; the problem adds its loss-weight -#: hash; the two national kernels share _NationalKernel's. -FULL_TARGETS = "uk.full.target_compilation@1, target_selection@1 and measures@1" -FULL_PROBLEM = "uk.full.problem@1" -NATIONAL = "uk.full.national_targets@1 and national_problem@1" -KERNEL_GROUPS = MappingProxyType( - { - FULL_TARGETS: tuple( - _KERNELS_BY_REF[ref] - for ref in ( - "uk.full.target_compilation@1", - "uk.full.target_selection@1", - "uk.full.measures@1", - ) - ), - FULL_PROBLEM: (_KERNELS_BY_REF["uk.full.problem@1"],), - NATIONAL: tuple(_KERNELS_BY_REF[ref] for ref in _NATIONAL_GRAPH), - } -) - -#: Imports a kernel's module makes only to build the hash, and the importers -#: the reason covers. They stay unhashed on purpose, so an edit to the list -#: re-keys only the kernels whose hashed modules it adds or removes. +#: Imports a kernel's module makes only to build an implementation hash, and +#: the importers the reason covers. The derived binding rule in test_uk_graph.py +#: leaves every implementation hash out of what a kernel's code computes +#: through, so such an import binds nothing. A target or problem kernel hashes +#: one only where another module it hashes imports it too, and none hashes the +#: first two, which stay unhashed on purpose: an edit to the list re-keys only +#: the kernels whose hashed modules it adds or removes +#: (test_a_target_kernel_hashes_a_hash_input_only_where_another_module_imports_it). HASH_INPUTS = MappingProxyType( { "microcosm.build.uk_runtime.target_compile_modules": HashInput( @@ -223,10 +210,11 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "parameter; no kernel's run reads it" ), ), - # graph_national imports these only inside the national gate and - # readback kernels' implementation hashes, which bind them there - # (microcosm#931); the national target and problem kernels never run - # those methods. + # graph_national imports these only inside implementation hashes, + # which bind them there (microcosm#931): the national gate and + # readback kernels', and for the last three the national problem's + # too, whose code computes through them. The national target and + # problem kernels never run the gate and readback kernels' methods. **{ module: HashInput( importers=("microcosm.build.uk_runtime.graph_national",), @@ -241,6 +229,19 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): "microcosm.build.trace", "microcosm.build.uk_runtime.donor_uprating", "microcosm.build.uk_runtime.etb_services", + ) + }, + **{ + module: HashInput( + importers=("microcosm.build.uk_runtime.graph_national",), + reason=( + "imported only in implementation hashes, to hash it there: " + "uk.full.national-gates@1's or uk.full.national-readback@1's, " + "and uk.full.national_problem@1's, whose code computes " + "through it" + ), + ) + for module in ( "microcosm.calibrate.monetary_binding", "microcosm.frame.adapters.policyengine_uk", "microcosm.frame.materialize", @@ -249,494 +250,6 @@ def test_national_target_kernels_hash_the_fact_decoder(monkeypatch): } ) -#: Every ``importer -> imported`` edge the walk from a kernel group's hashed -#: modules meets that the group does not hash and no review has classified, -#: with the groups that leave it unhashed. Recorded as found when the -#: per-kernel check was added; microcosm#1497 classifies them. Hash an entry -#: in the kernels that run it, move what they run into a module they hash, or -#: record why they need not hash it, then delete it here. The check fails on -#: an edge a group meets that is not recorded for it and on an edge recorded -#: for a group that no longer meets it, so the mapping can only shrink, and -#: removing a module from one group's hash fails even where another group -#: records the same edge. -UNCLASSIFIED_IMPORTS = MappingProxyType( - { - "microcosm.build.uk_runtime.graph_national -> microcosm.build.artifact_files": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.country_spec": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.country_spec": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.gate_battery": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.fiscal_targets -> microcosm.build.gates": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.gates": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.holdout": ( - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.input_mass": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.logbook_adoption": ( - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.plan": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.source_manifest": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.source_manifest": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.spec_engine.canonical": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.spec_engine.engine_abi": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.spec_engine.loader": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.spec_engine.model": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.spec_engine.schemas": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.spec_engine.take_up_semantics": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.spec_engine.yaml12": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.staging_v2": ( - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.build.trace": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.atomic_area_support": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.battery_bindings": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.calibration_run": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.cgt_projection": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.content_identity": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.dataset_size": ( - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.dfe_spring_term": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.diagnostics": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.frs_annual_income": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.uk_runtime.frs_spine": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.frs_spine": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.full_gates": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.local_authority_input -> microcosm.build.uk_runtime.geography_sources": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_build": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_calibration": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_population": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.graph_population": ( - FULL_TARGETS, - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_targets": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.graph_terminal": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.hmrc_replay -> microcosm.build.uk_runtime.hmrc_income": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.hmrc_income": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.uk_runtime.hmrc_source_contract": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.income_tax_rate_classes": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.lcfs_consumption": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.local_geography": ( - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.local_rowwise": ( - FULL_TARGETS, - NATIONAL, - ), - "microcosm.build.uk_runtime.full_problem -> microcosm.build.uk_runtime.local_rowwise": ( - FULL_TARGETS, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.local_rowwise": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.local_rowwise": ( - FULL_TARGETS, - ), - "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.national_calibration": ( - FULL_TARGETS, - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.national_doctrine": ( - FULL_TARGETS, - ), - "microcosm.build.uk_runtime.local_doctrine -> microcosm.build.uk_runtime.national_doctrine": ( - FULL_TARGETS, - ), - "microcosm.build.uk_runtime.frs_relationships -> microcosm.build.uk_runtime.national_frame": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.national_frame": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.national_frame": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.national_frame": ( - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.national_frame": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.weighted_integrity -> microcosm.build.uk_runtime.national_frame": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - # The target kernels hash receipt_counts beside weighted_integrity - # (8ea0ee182), and its floors read true sources through the shared - # lineage reader on the UK's declaration, the national schema and the - # shared source column (microcosm#931). The move's derived binding - # rule replaces this record. - "microcosm.build.uk_runtime.receipt_counts -> microcosm.build.source_lineage": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.receipt_counts -> microcosm.build.uk_runtime.household_lineage": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.receipt_counts -> microcosm.build.uk_runtime.national_frame": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.receipt_counts -> microcosm.frame.concepts": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.full_measure -> microcosm.build.uk_runtime.rowwise_dataset": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.rowwise_posture": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.size_checkpoint": ( - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.local_rowwise -> microcosm.build.uk_runtime.solve_progress": ( - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.build.uk_runtime.target_weights": ( - FULL_TARGETS, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.build.uk_runtime.uc_target_measurements": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.build.uk_runtime.weight_year": ( - NATIONAL, - ), - "microcosm.build.gates -> microcosm.build.us_runtime.stacked_spine": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.calibrate.solve -> microcosm.calibrate.exact_k": (FULL_PROBLEM,), - "microcosm.calibrate.solve -> microcosm.calibrate.gates": (FULL_PROBLEM,), - "microcosm.calibrate.solve -> microcosm.calibrate.initialization": ( - FULL_PROBLEM, - ), - "microcosm.calibrate.matrix -> microcosm.calibrate.monetary_binding": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.gates -> microcosm.calibrate.solve": (FULL_TARGETS, NATIONAL), - # Reviewed (round 2 of #1493): only national_calibration_manifest calls - # it, in uk.full.national-gates@1, which reads the solve's result. - "microcosm.build.uk_runtime.graph_national -> microcosm.calibrate.solve": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.calibrate.solve": ( - FULL_TARGETS, - ), - "microcosm.calibrate.artifacts -> microcosm.calibrate.solve": ( - FULL_TARGETS, - NATIONAL, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.adapters.policyengine_uk": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.national_frame -> microcosm.frame.adapters.policyengine_uk": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.country_spec -> microcosm.frame.bundle": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.frs_relationships -> microcosm.frame.bundle": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.full_measure -> microcosm.frame.bundle": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.frame.bundle": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.local_rowwise -> microcosm.frame.bundle": ( - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.bundle": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.national_calibration -> microcosm.frame.bundle": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.national_frame -> microcosm.frame.bundle": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.weighted_integrity -> microcosm.frame.bundle": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.calibrate.artifacts -> microcosm.frame.bundle": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.calibrate.matrix -> microcosm.frame.bundle": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.calibrate.solve -> microcosm.frame.bundle": (FULL_PROBLEM,), - "microcosm.build.uk_runtime.national_frame -> microcosm.frame.materialize": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.national_frame -> microcosm.frame.schema": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.frame.bundle -> microcosm.frame.schema": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.gates -> microcosm.frame.weights": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.full_measure -> microcosm.frame.weights": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.frame.weights": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.local_rowwise -> microcosm.frame.weights": ( - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.measure_simulation -> microcosm.frame.weights": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.national_calibration -> microcosm.frame.weights": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.national_frame -> microcosm.frame.weights": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.calibrate.artifacts -> microcosm.frame.weights": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.calibrate.matrix -> microcosm.frame.weights": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.calibrate.solve -> microcosm.frame.weights": (FULL_PROBLEM,), - "microcosm.frame.bundle -> microcosm.frame.weights": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.graph.canonical": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.canonical": ( - FULL_TARGETS, - FULL_PROBLEM, - ), - "microcosm.calibrate.artifacts -> microcosm.graph.canonical": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.codecs": ( - FULL_TARGETS, - FULL_PROBLEM, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.graph.decl": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.decl": ( - FULL_TARGETS, - FULL_PROBLEM, - ), - "microcosm.calibrate.artifacts -> microcosm.graph.decl": ( - FULL_TARGETS, - FULL_PROBLEM, - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_national -> microcosm.graph.kernel": ( - NATIONAL, - ), - "microcosm.build.uk_runtime.graph_targets -> microcosm.graph.kernel": ( - FULL_TARGETS, - FULL_PROBLEM, - ), - } -) - @pytest.mark.parametrize( ("kernel", "module"), @@ -819,18 +332,38 @@ def test_the_national_kernels_leave_calibrate_solve_unhashed(): assert calibrate_solve.__name__ not in hashed_module_names(kernel, module) -def test_every_import_a_target_kernel_reaches_is_hashed_or_recorded(): - """The walk covers each kernel's own module and every module hashed - beside the shared list, not only the list (microcosm#1492 review), and - the record is kept per group of kernels sharing one hash.""" - assert {kernel for group in KERNEL_GROUPS.values() for kernel, _ in group} == { - kernel for kernel, _ in _TARGET_AND_PROBLEM_KERNELS - } - mismatches = recorded_import_mismatches( - KERNEL_GROUPS, - listed=UK_TARGET_COMPILE_MODULE_NAMES, - excluded=frozenset(EXCLUDED_DIRECT_IMPORTS), - hash_inputs=HASH_INPUTS, - recorded=UNCLASSIFIED_IMPORTS, +def test_a_target_kernel_hashes_a_hash_input_only_where_another_module_imports_it(): + """What the derived binding rule does not cover (microcosm#931 ruling 6). + + The rule in test_uk_graph.py holds each target and problem kernel's hash + to what its code computes through, in place of microcosm#1497's per-kernel + import record, and leaves every implementation hash out of that code, so + an import made only to build a hash binds nothing. It does not check the + other side, that such a module stays out of the kernels' hashes. Each hash + input is still imported by every importer listed for it, and a target or + problem kernel hashes one only where a module it hashes, other than those + importers, imports it too, as measure_simulation, which the measures + compute through, imports the engine adapter. So the compile list and the + solve fingerprint's module stay unhashed, and an edit to a name or a + reason in either re-keys none of these kernels.""" + for name, hash_input in HASH_INPUTS.items(): + for importer in hash_input.importers: + assert name in first_party_imports(importer), (importer, name) + for kernel, module in _TARGET_AND_PROBLEM_KERNELS: + hashed = hashed_module_names(kernel, module) + for name, hash_input in HASH_INPUTS.items(): + if name not in hashed: + continue + beside = { + other + for other in hashed - set(hash_input.importers) + if name in first_party_imports(other) + } + assert beside, (kernel.ref, name) + hashed = set().union( + *(hashed_module_names(*pair) for pair in _TARGET_AND_PROBLEM_KERNELS) ) - assert not mismatches, mismatches + assert not hashed & { + "microcosm.build.uk_runtime.target_compile_modules", + "microcosm.build.uk_runtime.solve_scope", + } From 9f5469c199e132fb047f534d0e72be237c7a247b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:50:12 +0000 Subject: [PATCH 039/108] Follow up the review of the 9 Nov ruling commits (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three reviews of 8ea25b96b, e46f54a92, aaa371a54, 021182dca and 08b437a8e found no must. This commit takes the shoulds and the nits that need no ruling. Five go to María: the engine walk's root (the methods the stages call or the whole class), the reach near a power of two, where the phase-code tables live and a stale-code check in the spine gates, the two bindings the target rule forces, and the transport compile kernels' canonical encoder. The stage-42 notes' narrowing waits for (b)'s next stage-16/17/42 regeneration, so H2 regenerates once. Gate batteries: - The phase-code texts said an edit only one phase computes through re-keys that phase alone. A phase's own key leaves out what only the other phase computes through, but key chaining still re-keys the other phase wherever a node upstream hashes the module. graph_evidence's and graph_terminal's docstrings and comments, the battery table's comment, the battery test's docstring and the changelog now say so and name where: stages before the assembled gate hash stochastic_assignment and take_up_contract, and of the 38 modules only the terminal phase computes through, nine never re-solve, on a bound spine or in the in-graph build, high_value_home_gates among them. - 021182dca put calibration_run, the national calibration runner, on the preflight's key, so in the in-graph build an edit to it re-solved, against microcosm#1492's design. The runner's names the preflight computes through, the spine and local gate scopes and the scoped gate manifest, move unchanged into a new gate_scopes module, which calibration_run re-exports. The full gates' shared hash binds gate_scopes in its place, and the bound spine, the national gates and the certification hash it too. The calibrated gates' terminal_code binds calibration_run, whose admin totals and CGT projection artifact their gate context is built with. A new test holds that an edit to the runner leaves both full gate kernels' hashes and the preflight's own code, and moves the terminal code. - The spine gates' enum gates resolve a domain no artifact supplies through the rules engine they are handed (PolicyEngineUKEngine's enum_domain). The battery walk now follows the engine each battery's kernel is built with wherever a phase's gates read it (_BATTERY_ENGINES: the spine build's rules engine, and the driver's coverage engine for the full gates), so the spine gates' shared hash binds the engine adapter. A test checks that each engine is the one its build hands the kernel, and the artifact that hands it on. The engine root: _engine_reach repeats the engine walk over the engine methods its own reach names outside the engine's classes until the set stops growing, so a helper the engine hands itself to is followed into the methods it calls on that argument. It adds no module today. The control test gains that case. Identity tool (tests and text only): - The premises test builds one wrapper set from all three stage modules and counts references to a helper or wrapper, not only calls, so a second correction through another module's wrapper, an alias or a partial fails it. - A test pins that the exact tier takes c from the equal-halves members. On a synthetic family a float above 900, one member's pair sum a float below c leaves the tool green, which the least pair sum over every member would not. - The tolerance policy, the docstrings and the changelog say the exact tier holds where every member that shows equal halves shares c, which is what the code checks; a member the split's own fallback corrected keeps its family on the bracket. The changelog says c can sit a few floats above 60 / K (at most 16 for K up to 15, by design arithmetic), and its Declarations bullet says what the tool reports of the selection recompute's finds and misses. Texts: fixture_stage_plan_inputs' docstring says H2 parity binds the graph's CREATE kernel to the root transform it adds; the gate registry stop's reason says the rule follows the gate kernels past it; the codecs test reads the national line's kernels from its graph, the calibrated kernel among them; test_uk_target_compile_modules' docstring states the narrower hash-input rule; docs/uk-full-build-graph.md names frame.materialize among the engine stages' bindings. Corrections to the earlier messages, for the PR body: in 021182dca's later-edits bullet, the chaining above, and in the in-graph build calibration_run (until this commit), parity_reference and release_identity joined the solve's key; in aaa371a54, the 16 engine stages call materialize and 6 of them variable_metadata, and on a bound spine 24 of the local line's 33 nodes re-keyed, from uk.full.expand.owned on; in e46f54a92, the exact tier's condition above and the snap's range. What moves: node keys only. No value, declaration or generated file moves, and neither the solve nor the country fingerprint does. - 77 of the 99 UK kernel hashes move: every spine stage and expand kernel, the create, identity and claim kernels and the seven dense population kernels (graph_kernels' docstring and graph_evidence); the spine gates (with the engine adapter); the preflight and calibrated gates (gate_scopes for calibration_run); the export, package, inspection, readback and rotated holdout kernels; the bound spine and the certification; and the national targets, problem, gates and readback (graph_national). The calibrated gates' terminal_code moves too. The calibration and size kernels, the full target kernels, the geography, property and private rent kernels and the four atomic geography kernels keep theirs. - So every node of every UK graph re-keys: the hermetic and gated spine (79 and 81), the in-graph build (113), the local line on a bound spine (33) and the national line (6). The transport kernels keep their hashes. - source_stages.json and the coverage manifest are current. uk_spine.json and the H2 fixture hold stage contract hashes and no declaration changed, so neither regenerates. Tests (engine-free, synthetic or code facts): 26 touched and related files pass (1,133 tests). Ruff is clean on the 13 edited Python files, and ci_test_plan verify is ok. The unmutated tree passes, and 19 mutations under scratch are each caught: - a second correction in the incidence clone by attribute, by bare name in its transform, through cgt_support's wrapper, an alias, a nested function, a lambda or a partial; - an adapter helper calling concept_mapping on the engine it is handed, and a one-pass engine reach; - the exact tier taking the least pair sum over every member; - the spine gates dropping the adapter, the model ignoring the engine, and either build handing its gates another engine; - calibration_run back in the full gates' shared hash, the preflight reading a runner scope, and the full gates or the bound spine dropping gate_scopes; - the national line's calibrated kernel hashing the codecs. Co-Authored-By: Claude Opus 5.5 --- .../931-cgt-copies-source-weight.changed.md | 6 +- changelog.d/931-uk-helper-bindings.changed.md | 14 +- docs/uk-full-build-graph.md | 2 +- .../build/uk_runtime/calibration_run.py | 135 +------ .../build/uk_runtime/full_certification.py | 8 +- .../microcosm/build/uk_runtime/full_gates.py | 2 +- .../microcosm/build/uk_runtime/gate_scopes.py | 132 +++++++ .../microcosm/build/uk_runtime/graph_build.py | 12 +- .../build/uk_runtime/graph_evidence.py | 62 ++- .../build/uk_runtime/graph_kernels.py | 12 +- .../build/uk_runtime/graph_national.py | 4 +- .../build/uk_runtime/graph_terminal.py | 51 ++- .../tests/engine_free/uk/test_uk_graph.py | 359 +++++++++++++++--- .../uk/test_uk_identity_stability_receipts.py | 122 ++++-- .../uk/test_uk_target_compile_modules.py | 58 ++- tools/verify_uk_identity_stability.py | 10 +- 16 files changed, 713 insertions(+), 276 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index 6ac7294a1e..25693cfae8 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,6 +1,6 @@ The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. -- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: for a weight a few ulps above a multiple of 60 (at K > 1, by more than the snap absorbs), or where an exact-total correction or the #970 anchor's and the residential arms' roundings moved a member. So neither the count check nor the family test recounts the fold. At any K, a family of more than three members is held exactly where at least four of its members show their two clone halves at equal weights and share one pair sum c, whatever the others carry: its stored count n must be the count of a weight w with fl(w / n) = c. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so one of those four or more is uncorrected, and equal halves mean the anchor and the arms left its pair sum at exactly w / n, so c is exactly w / n. The stage counts n on each clone's own weight and gives every member fl(w / n), and the snapped count still rises with w, so this holds at K > 1 as at K = 1. Where every member shares c but fewer than four show equal halves, the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member where the floats either side of c are as wide, and two floats where c sits within two floats of a power of two, below which they are half as wide; c lies in (45 / K, 60 / K], which holds 16 at K = 3, 8 at K = 6 and 7, and 4 at K = 12 to 15. Every other family keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. The family test, which decides whether a household is a family at all, is held the same way at the K-aware count: a household the rule selects is possibly a family where its count at the folded weight times 1 + 1e-9 exceeds one and possibly whole where its count at the folded weight times 1 − 1e-9 is one, its stored state accepted in between, and the hold's counts decide where it applies. That bracket opens for a household whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies, a whole household or a family of two among them. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a household the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to a few floats next to the count's step, for some counts n. At K = 1 the step sits a float above the multiple, which leaves at most the multiple itself where the family is held exactly, or the multiple and the float either side of it where the hold reaches one float. At K from 2 to 15 the snap puts the step 3 to 17 floats above the multiple of 60 / K, which leaves at most the float on either side of the step where the family is held exactly, or up to three floats from it in the one-float tier, so a family on the multiple itself split into one copy more is reported. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. -- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception and no more, since it makes no one-copy check. Both stages' notes say where the correction lands, and the CGT stage's add that the shared fallback does not make the one-copy check, and what the identity tool's selection recompute can then report. The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. +- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: for a weight a few ulps above a multiple of 60 (at K > 1, by more than the snap absorbs), or where an exact-total correction or the #970 anchor's and the residential arms' roundings moved a member. So neither the count check nor the family test recounts the fold. At any K, a family of more than three members is held exactly where at least four of its members show their two clone halves at equal weights and every member that does shares one pair sum c, whatever the others carry: its stored count n must be the count of a weight w with fl(w / n) = c. A member that took the split's own fallback correction keeps equal halves at another weight, since the incidence clone halves it exactly, so its family keeps the bracket. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so one of those four or more is uncorrected, and equal halves mean the anchor and the arms left its pair sum at exactly w / n, so c is exactly w / n. The stage counts n on each clone's own weight and gives every member fl(w / n), and the snapped count still rises with w, so this holds at K > 1 as at K = 1. Where every member shares c but fewer than four show equal halves, the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member where the floats either side of c are as wide, and two floats where c sits within two floats of a power of two, below which they are half as wide; c lies in (45 / K, 60 / K] to within the snap, which can leave it a few floats above 60 / K (at most 16 for K up to 15, by design arithmetic over weights at and up to 40 floats above each multiple m × 60 / K, m from 4 to 3,999), and that range holds 16 at K = 3, 8 at K = 6 and 7, and 4 at K = 12 to 15. Every other family keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. The family test, which decides whether a household is a family at all, is held the same way at the K-aware count: a household the rule selects is possibly a family where its count at the folded weight times 1 + 1e-9 exceeds one and possibly whole where its count at the folded weight times 1 − 1e-9 is one, its stored state accepted in between, and the hold's counts decide where it applies. That bracket opens for a household whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies, a whole household or a family of two among them. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a household the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to a few floats next to the count's step, for some counts n. At K = 1 the step sits a float above the multiple, which leaves at most the multiple itself where the family is held exactly, or the multiple and the float either side of it where the hold reaches one float. At K from 2 to 15 the snap puts the step 3 to 17 floats above the multiple of 60 / K, which leaves at most the float on either side of the step where the family is held exactly, or up to three floats from it in the one-float tier, so a family on the multiple itself split into one copy more is reported. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. +- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception and no more, since it makes no one-copy check. Both stages' notes say where the correction lands, and the CGT stage's add that the shared fallback does not make the one-copy check, and that the identity tool's selection recompute can then find a family the stage never made, or miss one it made. The tool reports a family it misses. One it finds that the stage never made, a one-copy household the fallback lifted across 60 / K, moves only `copies_recomputed`, since the bracketed family test cannot tell it from a whole household (see Receipt and gate). The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. - **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy can carry it only where its corrected weight keeps its one copy and the walk's selection, the one-copy check `copy_rule` names, kept as a reading of the whole-households-first rule, so wherever one of the 64 carries it, the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the rule rerun on the stage's output would have counted a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy, then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. The fallback does not consult that check, so its row can also be a whole household, among them a one-copy household the check passed over; the rerun can then divide a different set. That can happen on synthetic frames, as it could under the rule before, with the weights the rule before gave: where the one-copy household sits one to three ulps below its band's support mass, the identity tool then reports a missing family, and where it weighs 60 at K = 1 and outweighs every copy, the rule rerun counts a family the stage never made, which the tool's family test, held to its bracket, does not report, since the corrected weight lies within a relative 1e-9 of 60. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate unless a selected household left at one copy outweighs it, or ties it from an earlier row, which needs every unselected household to weigh at most 60 / K (to within the K-ulp snap at K > 1). The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. A selected household left at one copy that can carry it also costs a rerun of the selection walk, about 0.41 s at that size, so 64 that the check passes over would take about 26 s; a national frame does not reach that, since its 64 heaviest whole households are unselected ones, far heavier than 60 / K. -- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot and a later one of the 64 can, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them; where none of the 64 can, nothing moves. At stage 42, and at K = 1 too, a selected household left at one copy that outweighs the heaviest unselected household, or ties it from an earlier row, comes first, which needs every unselected household to be as light; where it can carry the correction and keep the split, it takes it even where the heaviest unselected household could have, so those two rows' weights move by rounding amounts. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the hold, the count bracket and the bracketed family test. Against the exact recount it replaces, it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count, nor a family missing, or extra, where a household's re-summed fold lands across the one-copy boundary (60 / K) from its stored state: on the synthetic frames, the one false missing family the full-chain probes found, a two-copy family a float above 60 that folds onto 60, and one false extra family at K = 2. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, where at least four of its members show equal halves sharing one weight, whatever the others carry: at K = 1 for all but 31 of the 9,995 counts n from 4 to 9,998, and at K from 2 to 15 for every count. Where fewer show equal halves but every member shares one weight, the hold reaches one float, which at K = 1 takes in the multiple itself for 6,393 of those counts, so such a family goes unreported there, as the full-chain test's 900 family does at one clone; at K > 1 it is reported. It no longer reports a family of three or fewer members, or one whose members fold to different weights without four equal halves among them, nor a household split into two copies at 60 / K, which the family test's bracket cannot tell from a whole one. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`, and node keys chain on their inputs' keys, so every spine node from stage 26 on, and every node downstream of the spine, re-keys with them. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. +- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot and a later one of the 64 can, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them; where none of the 64 can, nothing moves. At stage 42, and at K = 1 too, a selected household left at one copy that outweighs the heaviest unselected household, or ties it from an earlier row, comes first, which needs every unselected household to be as light; where it can carry the correction and keep the split, it takes it even where the heaviest unselected household could have, so those two rows' weights move by rounding amounts. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the hold, the count bracket and the bracketed family test. Against the exact recount it replaces, it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count, nor a family missing, or extra, where a household's re-summed fold lands across the one-copy boundary (60 / K) from its stored state: on the synthetic frames, the one false missing family the full-chain probes found, a two-copy family a float above 60 that folds onto 60, and one false extra family at K = 2. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, where at least four of its members show equal halves and every member that does shares one weight, whatever the others carry: at K = 1 for all but 31 of the 9,995 counts n from 4 to 9,998, and at K from 2 to 15 for every count. Where fewer show equal halves but every member shares one weight, the hold reaches one float, which at K = 1 takes in the multiple itself for 6,393 of those counts, so such a family goes unreported there, as the full-chain test's 900 family does at one clone; at K > 1 it is reported. It no longer reports a family of three or fewer members, or one whose members fold to different weights without four equal halves among them, nor a household split into two copies at 60 / K, which the family test's bracket cannot tell from a whole one. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`, and node keys chain on their inputs' keys, so every spine node from stage 26 on, and every node downstream of the spine, re-keys with them. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 9d7ac4c1f5..50150467e2 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -21,7 +21,7 @@ Every UK spine stage now hashes each first-party module its transform computes t - `SOURCE_HOUSEHOLD_ID_COLUMN` from `microcosm.frame.concepts`, CopyLineage's default source column, the second listed name. The support channel writes the column by its literal name. Every stage that reads the constant declares the column by that name, which its key hashes, and counts before the copies only beside a copy column, or declares neither, so a renamed constant leaves the lineage readers to refuse or count as before. Still outside the rule: calls through any other object whose class the code never names, a name looked up by a computed string, and data files read by path. -- **The engine.** The 16 stages that call the engine compute through the `PolicyEngineUKEngine` the builds inject, whose class no stage's transform names. As María ruled on 9 Nov, a second walk is rooted at that class, the one `graph_kernels`' hermetic H2 builder and the spine build both construct (checked). It follows the methods a stage's code names on the engine, by attribute on any receiver or through `getattr`, with the constructor and every method they call on `self`, at the same scope and with the same tables, and the stage rule's tests hold what it reaches as part of the stage's walk. It matches methods by name, as the engine-call scan does, so it can only over-count. The engine stages call `materialize`, `variable_metadata` and, through `getattr` in `private_school_attendance`, `parameters` and `simulation_batches`. The walk reaches the three hand-bound engine modules and one more, `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. That module is the new `_ENGINE_COMPUTE_MODULES` group in `graph_kernels`, which every engine stage binds and which a test derives exactly. It sits beside `_ENGINE_MODULES`, which no other stage binds, because the SPI spine's three stages and the band donors bind `frame.materialize` for their own tables; any other stage binds it only where its own transform computes through it (checked). The engine reaches the concept vocabulary (`policyengine_uk_concepts`, `concept_mapping` and `concepts`) and `frame.rules` only through methods no engine stage calls (`concept_mapping` and `export_contract`), so none is bound for the engine. No spine stage hashes the concept vocabulary, and only `uc_take_up` and `pension_credit_take_up` hash `frame.rules`, for the protocol they type their engine by. The hermetic fixture's transform resolver no longer names the engine as held. +- **The engine.** The 16 stages that call the engine compute through the `PolicyEngineUKEngine` the builds inject, whose class no stage's transform names. As María ruled on 9 Nov, a second walk is rooted at that class, the one `graph_kernels`' hermetic H2 builder and the spine build both construct (checked). It follows the methods a stage's code names on the engine, by attribute on any receiver or through `getattr`, with the constructor and every method they call on `self`, and then the methods its own reach names on any receiver outside the engine's classes, as a helper the engine hands itself to might call, to a fixed point, at the same scope and with the same tables. The stage rule's tests hold what it reaches as part of the stage's walk. It matches methods by name, as the engine-call scan does, so it can only over-count. The engine stages call `materialize`, `variable_metadata` and, through `getattr` in `private_school_attendance`, `parameters` and `simulation_batches`. The walk reaches the three hand-bound engine modules and one more, `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. That module is the new `_ENGINE_COMPUTE_MODULES` group in `graph_kernels`, which every engine stage binds and which a test derives exactly. It sits beside `_ENGINE_MODULES`, which no other stage binds, because the SPI spine's three stages and the band donors bind `frame.materialize` for their own tables; any other stage binds it only where its own transform computes through it (checked). The engine reaches the concept vocabulary (`policyengine_uk_concepts`, `concept_mapping` and `concepts`) and `frame.rules` only through methods no engine stage calls (`concept_mapping` and `export_contract`), so none is bound for the engine. No spine stage hashes the concept vocabulary, and only `uc_take_up` and `pension_credit_take_up` hash `frame.rules`, for the protocol they type their engine by. The hermetic fixture's transform resolver no longer names the engine as held. - **What is bound.** `national_frame` joins the modules every spine stage hashes. The stage helper table gains 231 bindings of 54 modules in 49 stages. Among them: - in the CGT and high-value stages: `property_localisation`, `cgt_structure`, `spi_support`'s exact-total helpers, `cgt_imputation`, `cgt_asset_type` and `advani_summers`; - `employer_pension_contributions` in `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`; @@ -50,18 +50,18 @@ Every UK spine stage now hashes each first-party module its transform computes t - the spine's identity and claim kernels: `national_frame`, and the claim kernel `graph_frames` too, through which it casts the cells it claims (see the bound spine); - the four atomic geography kernels' shared hash: the content store, which `load_source_bytes` takes, and `frame.materialize`; - the seven calibration and size kernels' shared hash: `national_frame`, `calibrate.hierarchy`, `calibrate.monetary_binding` and the canonical encoder. The size search and refit hash `target_weights`, `local_doctrine`, `local_targets`, `national_doctrine` and `calibrate.geography_constants` too, in their own implementation hashes, since the dense solve and the other kernels sharing that hash do not compute through them; - - the bound spine: `frame.materialize`, the engine adapter, `content_identity`, `hmrc_uprating`, `gate_battery`, `gates`, `trace`, `graph_frames` and the canonical encoder. The create-frame normalisation it shares with the spine's root, and the owned-cell cast the spine's kernels share, moved unchanged from `graph_kernels` into the new `graph_frames`, which the spine's create, claim and stage kernels hash, so an edit to the spine's kernel module no longer re-keys the national line; + - the bound spine: `frame.materialize`, the engine adapter, `content_identity`, `hmrc_uprating`, `gate_battery`, `gates`, `trace`, `gate_scopes`, `graph_frames` and the canonical encoder. The create-frame normalisation it shares with the spine's root, and the owned-cell cast the spine's kernels share, moved unchanged from `graph_kernels` into the new `graph_frames`, which the spine's create, claim and stage kernels hash, so an edit to the spine's kernel module no longer re-keys the national line; - the export description: `gates`, `atomic_area_support`, `graph_population`, `hmrc_uprating`, `local_authority_input`, `rowwise_dataset`, `rowwise_geography`, `frame.materialize` and the canonical encoder. The inspection export binds the same modules, and `geography_ladder`, `national_frame` and `weight_year` too; - the export readback: `artifact_files`, `trace`, `geography_ladder`, the engine adapter, `frame.materialize` and the canonical encoder; the package inventory: `artifact_files`, `trace` and the canonical encoder; - - the certification: `artifact_files`, `trace`, `gate_battery`, `gates`, `graph_evidence`, `calibration_run` and the canonical encoder; + - the certification: `artifact_files`, `trace`, `gate_battery`, `gates`, `graph_evidence`, `gate_scopes` (the spine gate manifest's scope) and the canonical encoder; - the rotated holdout: the 35 modules that its problem reconstruction, folds and rowwise solves compute through. They are the local surface's modules, `holdout`, `local_target_census`, `size_checkpoint`, `solve_progress`, `weighted_integrity`, microcosm.calibrate's solver modules, `gate_battery`, `stage_evidence`, the engine adapter and the canonical encoder; - - the national gates: `gates`, microcosm.calibrate's `artifacts`, `matrix`, `registry`, `target`, `hierarchy`, `monetary_binding` and `solve`, `graph_targets`, `graph_population`, `national_frame`, `etb_services`, `cgt_projection`, `donor_uprating` and the canonical encoder, and everything the gate registry's bindings compute through (see Gate batteries); + - the national gates: `gates`, microcosm.calibrate's `artifacts`, `matrix`, `registry`, `target`, `hierarchy`, `monetary_binding` and `solve`, `graph_targets`, `graph_population`, `national_frame`, `etb_services`, `cgt_projection`, `donor_uprating`, `gate_scopes` and the canonical encoder, and everything the gate registry's bindings compute through (see Gate batteries); - the national readback: `artifact_files`, `trace`, `frame.materialize`, the engine adapter, `geography_ladder`, `graph_population`, `hmrc_uprating` and the canonical encoder. The LAD23 names' pin stays listed, now for every kernel that reads it. The geographic clone index columns' name (`rowwise_dataset.ladder_clone_index_column`) is listed for the calibrated and national gates, whose support-family fold reads it: only the dense expand writes those columns, and it hashes `rowwise_dataset`; the national frame has no ladder, and the fold keys on the column only where the table carries it. It is listed for the dense measures and the national problem too (see Target and problem kernels). `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside implementation hashes: the national gates' and readback's, and for three of them the national problem's. -- **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Another test checks each part of that model against the code. So an edit only the transferred phase computes through re-keys neither the assembled gate nor the spine stages that wait on it, and one only the terminal phase computes through re-keys neither the preflight nor the dense solve keyed on it. - - The spine gates' hash binds the 11 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants` and `calibrate.geography_constants`, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. As before, the spine gate kernel does not check its phase's code. - - The full gates' hash binds the 26 modules both phases compute through. It loses `local_rowwise`, `weighted_integrity` and `diagnostics`, which only the terminal phase computes through. The preflight node carries a new `preflight_code`: the shared ledger's registry parity, behind the ledger compile parity gates only the preflight declares. The calibrated node's `terminal_code` now hashes 37 modules. `high_value_homes` and `calibrate.geography_constants` move into the shared hash, since the preflight computes through them too; the target kernels hash both, so an edit to either already re-solved. The kernel refuses a stale or missing `preflight_code`, as it refuses a stale `terminal_code`. +- **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Where a phase's gates read the engine their kernel hands them, the walk also follows the methods they call on it, from the engine's class, as for an engine stage (see The engine): the spine gates' rules engine is the spine build's `PolicyEngineUKEngine`, and the full gates' engine is the driver's coverage engine. Another test checks each part of that model against the code, the engines among them. So the assembled gate's own key, which the spine stages after it chain on, carries nothing only the transferred phase computes through, and the preflight's, which the dense solve chains on, nothing only the terminal phase does. Key chaining still re-keys them wherever a node upstream hashes the edited module. Of the 17 modules only the transferred phase computes through, stages at or before `frs_brma`, the assembled phase's last, hash `stochastic_assignment` and `take_up_contract`, so an edit to either re-keys the assembled gate; no stage before it hashes the other 15. Of the 38 only the terminal phase computes through, most re-solve too, through the spine stages, the target and problem kernels, the population kernels or the dense solve, which hash them as well. Nine never re-solve, on a bound spine or in the in-graph build: `high_value_home_gates`, `uk_runtime.diagnostics`, `microcosm.diagnostics`' catalogs and schema, `calibrate.hierarchy_labels`, the target loss attribution, `input_mass`, `cgt_projection` and `source_runtime`, so a disclosure fix between two builds that must reuse the solve still belongs in `high_value_home_gates`. `calibration_run` and `release_certification` re-solve only on a bound spine, whose kernel hashes both, and seven modules that only the spine's nodes hash before the solve re-solve only in the in-graph build. The preflight's own code, the shared ledger's parity report, is also hashed by the full build's target compilation, selection, measures and problem, by the national line's targets, problem and gates and, in the in-graph build, by the `etb_services` and `lcfs_consumption` stages and the transferred gate, so an edit to it re-keys those too. + - The spine gates' hash binds the 12 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants`, `calibrate.geography_constants` and the engine adapter, through whose `PolicyEngineUKEngine.enum_domain` each phase's enum gates resolve a domain no artifact supplies, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. As before, the spine gate kernel does not check its phase's code. + - The full gates' hash binds the 26 modules both phases compute through. It loses `local_rowwise`, `weighted_integrity` and `diagnostics`, which only the terminal phase computes through. It binds the spine and local gate scopes and the scoped gate manifest, which the preflight computes through, in a new `gate_scopes` module, not in `calibration_run`, the national calibration runner they lived in. They moved there unchanged, and `calibration_run` re-exports them, as microcosm#1492 moved the target kernels' two functions into `ledger_surface`, so in the in-graph build an edit elsewhere in the runner re-keys neither the preflight nor the solve. The bound spine, the national gates and the certification, whose code reaches them too, hash `gate_scopes` as well. The calibrated gates' `terminal_code` binds `calibration_run` itself, for the admin totals and CGT projection artifact their gate context is built with. The preflight's release input coverage gate computes through `stage_evidence`, `parity_reference`, `release_identity` and `release_input_coverage`, which its key used to miss, so on a bound spine an edit to any of them now re-keys the preflight and re-solves; in the in-graph build so does one to `parity_reference` or `release_identity`, while the other two already re-solved through the spine stages that hash them. The preflight node carries a new `preflight_code`: the shared ledger's registry parity, behind the ledger compile parity gates only the preflight declares. The calibrated node's `terminal_code` now hashes 38 modules. `high_value_homes` and `calibrate.geography_constants` move into the shared hash, since the preflight computes through them too; the target kernels hash both, so an edit to either already re-solved. The kernel refuses a stale or missing `preflight_code`, as it refuses a stale `terminal_code`. - The national gates evaluate whichever gates their node's scope names, so their hash binds everything the gate registry's bindings compute through. That is 50 modules, which `battery_bindings.UK_GATE_REGISTRY_MODULES` lists and a test holds exactly to the registry's derived reach. Their stop at the registry, the one held kernel stop, goes. - **Target and problem kernels.** As María ruled on 9 Nov, the derived rule replaces microcosm#1497's per-kernel import record for the full build's target compilation, selection, measures and problem and the national targets and problem, which that record held until now. No kernel stays held. - `test_uk_target_compile_modules` loses the record (`UNCLASSIFIED_IMPORTS`), its kernel groups and the check that read them. It keeps the shared list's closure check, the tests of what the kernels hash and its hash inputs, the seven above among them. A new check covers what the derived rule does not: each hash input is still imported where listed, and a kernel hashes one only where another module it hashes imports it too. So no kernel hashes the compile list or the solve fingerprint's module, and an edit to either re-keys none. diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index 8b7e22bf64..bd1c211695 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -114,7 +114,7 @@ The full target compiler preserves the unreduced band-edge register, reference-p A spine engine call whose frame carries more than one clone runs one simulation per clone, since one simulation over a cloned spine does not fit in memory. `PolicyEngineUKEngine.materialize` decides from the frame, not from a setting: a call whose household table carries `household_clone_index` with more than one clone runs its simulations in ascending clone order, and each is released before the next is built. A frame without the column, or with one clone, takes one simulation as before. A batched call's requests are judged first. A period in the rebalancing years, a refused formula, a frame without `months_since_last_birthday`, or a request for `attends_private_school`, `private_school_vat`, `baseline_private_school_vat` or `gov_tax` before stage 52 stores attendance refuses the call before any engine is built. Until then the spine's pass lets attendance run only where the three parameters the private-school VAT rate reads equal their baseline's and `private_school_vat_change` comes out zero, and only if `gov_tax` does not run. Unlike a measure block, each batch reads the pool's shares at the frame's own weights, unscaled (`c_b = S_b/S`), and is judged under the spine's pass, which raises on any violation rather than recording it, since the values come back as plain arrays. People come back in the frame's order and groups in ascending id order, as one simulation returns them. No spine frame carries `household_clone_index` yet, so every spine call takes one simulation today, and batching moves no spine value. -policyengine-uk's `attends_private_school` ranks each household's market income plus benefits, less its income tax and National Insurance, over the people a simulation holds, so stage 52 (`private_school_attendance`) restates the formula's last step over the whole frame. It computes that income from household-local engine values, ranks it with microdf's `MicroSeries.percentile_rank`, and reads the attendance rates and population factor from the engine's own parameters. Over one simulation it also asks for the engine's flags, stores them once the restatement matches them bit for bit, and raises otherwise. Clone by clone it asks only for the household-local values and stores the restatement. Its evidence records `rank_route` (`engine_with_parity` or `pooled_restatement`) and `engine_parity_checked`. The spine sidecar's `rules_engine` gains `engine_batches` only on a spine whose calls run clone by clone, and the engine identity does not move with batching. Every stage that calls the engine through the adapter binds the adapter, `engine_population` and `policyengine_uk_population` into its implementation hash. +policyengine-uk's `attends_private_school` ranks each household's market income plus benefits, less its income tax and National Insurance, over the people a simulation holds, so stage 52 (`private_school_attendance`) restates the formula's last step over the whole frame. It computes that income from household-local engine values, ranks it with microdf's `MicroSeries.percentile_rank`, and reads the attendance rates and population factor from the engine's own parameters. Over one simulation it also asks for the engine's flags, stores them once the restatement matches them bit for bit, and raises otherwise. Clone by clone it asks only for the household-local values and stores the restatement. Its evidence records `rank_route` (`engine_with_parity` or `pooled_restatement`) and `engine_parity_checked`. The spine sidecar's `rules_engine` gains `engine_batches` only on a spine whose calls run clone by clone, and the engine identity does not move with batching. Every stage that calls the engine through the adapter binds the adapter, `engine_population` and `policyengine_uk_population` into its implementation hash, and `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. ## Replay and outputs diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 3d1406da30..96629d011b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -5,11 +5,13 @@ shared executable graph (:mod:`.graph_national`, the driver's national path in :mod:`.full_build_cli`). What stays here is what both lines and the release-cut certifier still consume: the Logbook pipeline and attempt-id -prefix, the four gate scopes and their closed-world exclusions, the bound -spine sidecar and checkpoint authentication, the spine provenance blocks, -the scoped gate manifest, the CGT projection and admin-anchor artifacts, -the Ledger and runtime provenance blocks, the register census and doctrine -payloads, and the scoped-report signing. +prefix, the gate scopes and their closed-world exclusions, the bound spine +sidecar and checkpoint authentication, the spine provenance blocks, the CGT +projection and admin-anchor artifacts, the Ledger and runtime provenance +blocks, the register census and doctrine payloads, and the scoped-report +signing. The spine and local gate scopes and the scoped gate manifest live +in :mod:`.gate_scopes`, which the full build's gate battery hashes for them +in place of this module, and are re-exported here. """ from __future__ import annotations @@ -47,6 +49,16 @@ UK_NHS_SPENDING_COMPONENT_COLUMNS, ) +# Moved to gate_scopes, which the full build's gate battery hashes, so an +# edit elsewhere in this module no longer re-keys its preflight or the solve +# keyed on it (microcosm#931, as microcosm#1492 moved the two functions +# below). Kept importable from here for this module's other callers. +from microcosm.build.uk_runtime.gate_scopes import ( + UK_LOCAL_GATE_SCOPE, + UK_SPINE_GATE_SCOPE, + uk_scoped_gate_manifest, +) + # Moved to ledger_surface, which the UK target kernels hash, so an edit # elsewhere in this module no longer re-keys them (microcosm#1492). Kept # importable from here for this module's other callers. @@ -87,16 +99,6 @@ "uk_calibration_reference_coverage", ) -UK_LOCAL_GATE_SCOPE = ( - "uk_local_geography_ladder_post_calibration", - "uk_local_brma_enum_domain", - "uk_local_area_support", - "uk_local_target_fit", - "uk_local_per_family_fit", - "uk_local_weight_ratio", - "uk_local_weight_ess", -) - #: Report-only gates the dense battery evaluates on the calibrated pool beside #: the local battery; they are excluded from certification like the local scope. UK_LOCAL_DIAGNOSTIC_GATE_SCOPE = ( @@ -104,79 +106,6 @@ "uk_high_value_homes_support", ) -UK_SPINE_GATE_SCOPE = ( - "uk_stage_was_wealth_support", - "uk_stage_was_wealth_coherence", - "uk_stage_was_lisa_support", - "uk_stage_was_lisa_anchor", - "uk_stage_high_value_home_support_mass", - "uk_stage_property_components", - "uk_stage_nts_bus_travel_support", - "uk_stage_nts_bus_travel_facts", - "uk_stage_uc_deduction_attributes", - "uk_stage_lcfs_consumption_support", - "uk_stage_lcfs_consumption_energy_rake", - "uk_stage_lcfs_consumption_bus_pricing", - "uk_stage_lcfs_consumption_road_fuel_incidence", - "uk_stage_lcfs_consumption_road_fuel_level", - "uk_stage_lcfs_consumption_recomposed_totals", - "uk_stage_lcfs_consumption_excise_quantities", - "uk_stage_lcfs_consumption_gambling_participation", - "uk_stage_lcfs_consumption_basis", - "uk_stage_etb_vat_support", - "uk_stage_etb_services_support", - "uk_stage_etb_services_support_pricing", - "uk_stage_frs_hmrc_spine_leaves_signal", - "uk_stage_frs_annual_income_spells", - "uk_stage_spi_support_channel_mass", - "uk_stage_hmrc_spi_income_spine_identity", - "uk_stage_hmrc_spi_income_spine_hourly_pay", - "uk_stage_spi_benefit_coherence", - "uk_stage_uc_take_up", - "uk_stage_pension_credit_take_up", - "uk_stage_child_benefit_take_up", - "uk_stage_cgt_incidence_clone_mass", - "uk_stage_cgt_support_split_mass", - "uk_stage_cgt_residential_split_mass", - "uk_stage_spi_income_band_donors_support", - "uk_stage_hmrc_cgt_gains_spine_summary", - "uk_stage_hmrc_cgt_asset_type_spine_summary", - "uk_stage_cgt_incidence_anchor_composition", - "uk_stage_salary_sacrifice_realization", - "uk_stage_student_loans_realization", - "uk_stage_age_tail_targets", - "uk_stage_frs_relationships_composition", - "uk_stage_regional_home_values_ranking", - "uk_stage_property_purchases_fit", - "uk_stage_uc_legacy_closure_take_up_contract", - "uk_stage_frs_person_draws_apprentices", - # The take-up flags at their drawing stages, before calibration (María's - # ruling of 2026-10-29, microcosm#1095). - "uk_stage_frs_take_up_signal", - "uk_stage_frs_person_draws_take_up_signal", - "uk_stage_frs_household_draws_take_up_signal", - "uk_stage_childcare_copy_take_up_signal", - "uk_stage_isa_subscriptions_invariants", - "uk_stage_isa_subscriptions", - "uk_stage_company_cars_invariants", - "uk_stage_spi_employment_shell_fit", - "uk_stage_aps_occupation_fit", - # Weight-independent like the BRMA enum below, and its column exists from - # frs_relationships onward (#791). - "uk_ons_household_type_enum_domain", - # Weight-independent; the column exists only from hmrc_cgt_asset_type_spine - # onward, so it is checked at the transferred boundary, and its domain is - # declared by the stage module (microcosm#725). - "uk_capital_gains_asset_type_enum_domain", - # Weight-independent; spi_housing_shell is the last spine stage that writes - # the band, so the transferred boundary checks every row's nation - # (microcosm#1278). - "uk_council_tax_band_domain", - # Weight-independent, and its column exists from frs_brma onward, so the - # spine checks it at the assembled boundary instead of the release end. - "uk_brma_enum_domain", -) - UK_NATIONAL_GATE_SCOPE = ( "uk_release_input_coverage_manifest_current", "uk_release_family_build_stages", @@ -618,36 +547,6 @@ def _calibration_gate_manifest() -> GatesManifest: ) -def uk_scoped_gate_manifest( - scope: tuple[str, ...] | frozenset[str], - *, - phases: tuple[str, ...], - policy_suffix: str, - source: GatesManifest | None = None, -) -> GatesManifest: - """Filter the declared UK gate spec to one battery's scope. - - The one scope-filtering implementation every scoped producer shares - (spine build, calibration seam, release cut). ``source`` lets a caller - that already holds a loaded spec — or a hermetic test that stubs one — - supply it; the default is the committed package spec. - """ - - if source is None: - source = load_country_spec("uk").gates - entries = tuple(entry for entry in source.gates if entry.id in scope) - missing = sorted(set(scope) - {entry.id for entry in entries}) - if missing: - raise RuntimeError(f"UK gate scope names undeclared gate id(s): {missing}.") - return GatesManifest( - country=source.country, - version=source.version, - policy=f"{source.policy}; {policy_suffix}", - phases=phases, - gates=entries, - ) - - #: Admin anchors published against a concept the frame carries only in parts. #: The anchor keeps the publisher's shape (one NHS budget line); our stages #: carry the spend split by point of delivery. Composing is the translation, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py index 26de2c75c0..d1143d6c13 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py @@ -402,13 +402,13 @@ def implementation_hash(self): from microcosm.data import contract from .. import artifact_files, gate_battery, gates, trace - from . import calibration_run, full_gates, graph_evidence, graph_terminal + from . import full_gates, gate_scopes, graph_evidence, graph_terminal # The readiness also computes through the comparison files' records # (artifact_files, trace's file digest), the phase reports it decodes # (gate_battery, gates), the spine gate manifest and its scope - # (graph_evidence, calibration_run) and its report's canonical - # encoding (microcosm#931). + # (graph_evidence, gate_scopes) and its report's canonical encoding + # (microcosm#931). return source_hash( sys.modules[__name__], contract, @@ -419,7 +419,7 @@ def implementation_hash(self): gate_battery, gates, graph_evidence, - calibration_run, + gate_scopes, canonical_json, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py index 6d3d201960..1752c803b0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py @@ -325,7 +325,7 @@ def classify_full_gate_outcomes( BLOCKS_ARTIFACT policy, including its declared missing-evidence rules. """ from microcosm.build.gate_battery import GatePhaseReport, GateStatus - from microcosm.build.uk_runtime.calibration_run import UK_LOCAL_GATE_SCOPE + from microcosm.build.uk_runtime.gate_scopes import UK_LOCAL_GATE_SCOPE if not isinstance(report, GatePhaseReport): raise TypeError("Classify a validated GatePhaseReport, not unbound JSON.") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py new file mode 100644 index 0000000000..f58e980c01 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py @@ -0,0 +1,132 @@ +"""The UK spine and local gate scopes, and the scoped gate manifest. + +All three lived in ``calibration_run``, the national calibration runner. The +full build's gate battery computes through them: both its phases build the +spine gate manifest (``graph_evidence.uk_spine_gate_manifest``) and classify +the local gates (``full_gates.classify_full_gate_outcomes``), and the dense +solve keys on the source preflight. So the battery's shared hash binds this +small module instead of all of ``calibration_run`` (microcosm#931), as the +target kernels hash ``ledger_surface`` (microcosm#1492), and an edit +elsewhere in the runner re-keys neither the preflight nor, in the in-graph +build, the solve keyed on it. The calibrated gates' own code still binds the +runner, whose admin totals and CGT projection artifact their gate context is +built with. ``calibration_run`` re-exports all three for its other callers, +beside the calibration, national and local diagnostic scopes it keeps. +""" + +from __future__ import annotations + +from microcosm.build.country_spec import GatesManifest, load_country_spec + +UK_LOCAL_GATE_SCOPE = ( + "uk_local_geography_ladder_post_calibration", + "uk_local_brma_enum_domain", + "uk_local_area_support", + "uk_local_target_fit", + "uk_local_per_family_fit", + "uk_local_weight_ratio", + "uk_local_weight_ess", +) + +UK_SPINE_GATE_SCOPE = ( + "uk_stage_was_wealth_support", + "uk_stage_was_wealth_coherence", + "uk_stage_was_lisa_support", + "uk_stage_was_lisa_anchor", + "uk_stage_high_value_home_support_mass", + "uk_stage_property_components", + "uk_stage_nts_bus_travel_support", + "uk_stage_nts_bus_travel_facts", + "uk_stage_uc_deduction_attributes", + "uk_stage_lcfs_consumption_support", + "uk_stage_lcfs_consumption_energy_rake", + "uk_stage_lcfs_consumption_bus_pricing", + "uk_stage_lcfs_consumption_road_fuel_incidence", + "uk_stage_lcfs_consumption_road_fuel_level", + "uk_stage_lcfs_consumption_recomposed_totals", + "uk_stage_lcfs_consumption_excise_quantities", + "uk_stage_lcfs_consumption_gambling_participation", + "uk_stage_lcfs_consumption_basis", + "uk_stage_etb_vat_support", + "uk_stage_etb_services_support", + "uk_stage_etb_services_support_pricing", + "uk_stage_frs_hmrc_spine_leaves_signal", + "uk_stage_frs_annual_income_spells", + "uk_stage_spi_support_channel_mass", + "uk_stage_hmrc_spi_income_spine_identity", + "uk_stage_hmrc_spi_income_spine_hourly_pay", + "uk_stage_spi_benefit_coherence", + "uk_stage_uc_take_up", + "uk_stage_pension_credit_take_up", + "uk_stage_child_benefit_take_up", + "uk_stage_cgt_incidence_clone_mass", + "uk_stage_cgt_support_split_mass", + "uk_stage_cgt_residential_split_mass", + "uk_stage_spi_income_band_donors_support", + "uk_stage_hmrc_cgt_gains_spine_summary", + "uk_stage_hmrc_cgt_asset_type_spine_summary", + "uk_stage_cgt_incidence_anchor_composition", + "uk_stage_salary_sacrifice_realization", + "uk_stage_student_loans_realization", + "uk_stage_age_tail_targets", + "uk_stage_frs_relationships_composition", + "uk_stage_regional_home_values_ranking", + "uk_stage_property_purchases_fit", + "uk_stage_uc_legacy_closure_take_up_contract", + "uk_stage_frs_person_draws_apprentices", + # The take-up flags at their drawing stages, before calibration (María's + # ruling of 2026-10-29, microcosm#1095). + "uk_stage_frs_take_up_signal", + "uk_stage_frs_person_draws_take_up_signal", + "uk_stage_frs_household_draws_take_up_signal", + "uk_stage_childcare_copy_take_up_signal", + "uk_stage_isa_subscriptions_invariants", + "uk_stage_isa_subscriptions", + "uk_stage_company_cars_invariants", + "uk_stage_spi_employment_shell_fit", + "uk_stage_aps_occupation_fit", + # Weight-independent like the BRMA enum below, and its column exists from + # frs_relationships onward (#791). + "uk_ons_household_type_enum_domain", + # Weight-independent; the column exists only from hmrc_cgt_asset_type_spine + # onward, so it is checked at the transferred boundary, and its domain is + # declared by the stage module (microcosm#725). + "uk_capital_gains_asset_type_enum_domain", + # Weight-independent; spi_housing_shell is the last spine stage that writes + # the band, so the transferred boundary checks every row's nation + # (microcosm#1278). + "uk_council_tax_band_domain", + # Weight-independent, and its column exists from frs_brma onward, so the + # spine checks it at the assembled boundary instead of the release end. + "uk_brma_enum_domain", +) + + +def uk_scoped_gate_manifest( + scope: tuple[str, ...] | frozenset[str], + *, + phases: tuple[str, ...], + policy_suffix: str, + source: GatesManifest | None = None, +) -> GatesManifest: + """Filter the declared UK gate spec to one battery's scope. + + The one scope-filtering implementation every scoped producer shares + (spine build, calibration seam, release cut). ``source`` lets a caller + that already holds a loaded spec — or a hermetic test that stubs one — + supply it; the default is the committed package spec. + """ + + if source is None: + source = load_country_spec("uk").gates + entries = tuple(entry for entry in source.gates if entry.id in scope) + missing = sorted(set(scope) - {entry.id for entry in entries}) + if missing: + raise RuntimeError(f"UK gate scope names undeclared gate id(s): {missing}.") + return GatesManifest( + country=source.country, + version=source.version, + policy=f"{source.policy}; {policy_suffix}", + phases=phases, + gates=entries, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py index 56cc3949e4..77a483554e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -393,20 +393,22 @@ def implementation_hash(self) -> str: from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from .. import gate_battery, gates, trace - from . import content_identity, graph_frames, hmrc_uprating + from . import content_identity, gate_scopes, graph_frames, hmrc_uprating # The node also computes through the H5 it loads (frame_materialize, # the engine adapter's contracts), the checkpoint's content identity # and pinned engine (content_identity, hmrc_uprating), the gate # digests it derives through a battery run (gate_battery, gates, - # trace), the create-frame normalisation it shares with the spine's - # root (graph_frames, not the spine's kernel module, so an edit there - # does not re-key the national line) and the canonical encoding of its - # artifacts (microcosm#931). + # trace) over the spine gate scope and its scoped manifest + # (gate_scopes), the create-frame normalisation it shares with the + # spine's root (graph_frames, not the spine's kernel module, so an + # edit there does not re-key the national line) and the canonical + # encoding of its artifacts (microcosm#931). return source_hash( type(self), national_frame, calibration_run, + gate_scopes, release_certification, frame_materialize, uk_engine_adapter, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py index 3830260865..3adf9d7660 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -44,9 +44,9 @@ encode_stage_evidence, ) from . import battery_bindings, stage_health -from .calibration_run import UK_SPINE_GATE_SCOPE, uk_scoped_gate_manifest from .cgt_asset_type import CGT_ASSET_TYPE_DOMAIN from .frs_relationships import CHRONICLE_ONS_HOUSEHOLD_TYPE_VALUE_IDS +from .gate_scopes import UK_SPINE_GATE_SCOPE, uk_scoped_gate_manifest from .graph import UK_SPINE_ASSEMBLED_FINAL_STAGE #: The spine checkpoint's build state the terminal gates read as the spine @@ -182,7 +182,10 @@ def uk_spine_assembled_only_code_hash() -> str: through: the ONS household type domain its enum gate checks (``frs_relationships``), which no transferred-phase gate reads. - Hashed once per process, as for the calibrated gates' own code + So the transferred phase's own key leaves it out, though an edit to it + still re-keys the transferred gate, through the ``frs_relationships`` + stage, whose module it is, and every stage after it. Hashed once per + process, as for the calibrated gates' own code (``graph_terminal.uk_terminal_only_code_hash``). """ from . import frs_relationships @@ -207,10 +210,18 @@ def uk_spine_transferred_only_code_hash() -> str: (``frs_annual_income``); the apprentice draw's take-up contract (``take_up_contract``); and the latent-attribute checks' binomial band (``stochastic_assignment``). Its - enum gate also checks the CGT asset type domain (``cgt_asset_type``). The - assembled gate, which every later spine stage waits on, therefore re-keys - on none of them (the re-review of microcosm#1420, E; microcosm#931 ruling - 6). Hashed once per process, as the assembled phase's own code is. + enum gate also checks the CGT asset type domain (``cgt_asset_type``). + + The assembled gate's own key, on which every later spine stage's chains, + therefore carries none of them (the re-review of microcosm#1420, E; + microcosm#931 ruling 6). An edit to one still re-keys the assembled gate + wherever a stage before it hashes the module: stages at or before + ``frs_brma``, the assembled phase's last, hash ``stochastic_assignment`` + and ``take_up_contract``, so an edit to either re-keys every spine node + after them, the assembled gate included. No stage before that gate hashes + the other fifteen, so an edit to one of those re-keys the stages that + hash it, those after them and the transferred gate, never the assembled + gate. Hashed once per process, as the assembled phase's own code is. """ from .. import chronicle_epoch, stochastic_assignment from .. import ledger_targets as shared_ledger_targets @@ -254,9 +265,11 @@ def uk_spine_transferred_only_code_hash() -> str: #: Each spine gate phase's own code: the hash of what only that phase's gates #: compute through, which its node carries as its ``_code`` parameter, -#: so an edit there re-keys that phase alone. The code both phases compute -#: through is the gate kernel's own hash (microcosm#931 ruling 6); -#: test_uk_graph derives each phase's share from the gates it declares. +#: so no other phase's own key carries it. An edit there still re-keys the +#: other phase wherever a node upstream of it hashes the module, as each +#: function's docstring says. The code both phases compute through is the +#: gate kernel's own hash (microcosm#931 ruling 6); test_uk_graph derives each +#: phase's share from the gates it declares. UK_SPINE_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( { "assembled": uk_spine_assembled_only_code_hash, @@ -439,10 +452,13 @@ class UKSpineGateKernel(KernelBase): ``synthetic_smoke`` mirrors ``GateBatteryRun(synthetic_smoke=...)`` — on a data-only fixture the population-fact gates are recorded but never block. Each phase also carries ``_code``, the hash of the code only its - own gates compute through (:data:`UK_SPINE_GATE_PHASE_CODE`), so an edit - there re-keys that phase alone: the assembled gate, which every later - spine stage waits on, re-keys on none of the transferred phase's (the - re-review of microcosm#1420, E; microcosm#931 ruling 6). + own gates compute through (:data:`UK_SPINE_GATE_PHASE_CODE`), so the + assembled gate's key, on which every later spine stage's chains, carries + none of the transferred phase's (the re-review of microcosm#1420, E; + microcosm#931 ruling 6). An edit to a module only the transferred phase + computes through still re-keys the assembled gate where a stage before it + hashes the module, as for ``stochastic_assignment`` and + ``take_up_contract``. """ ref = "uk.spine-gates@1" @@ -460,6 +476,7 @@ def __init__(self, *, gates: GatesManifest, engine: object, engine_identity: str def implementation_hash(self) -> str: import microcosm.calibrate.geography_constants as geography_constants + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from microcosm.graph.canonical import canonical_json from .. import gates, stage_evidence @@ -481,12 +498,18 @@ def implementation_hash(self) -> str: # battery and the gate registry (gate_battery, battery_bindings, # gates); the stage-health checks (stage_health, the independent # review of microcosm#1420, finding 5), with the receipt bands of the - # take-up checks both phases declare (receipt_counts); and the enum - # gate's nation partition (council_tax_constants, the region tier). - # What only one phase's gates compute through is that phase's own - # code, a node param (UK_SPINE_GATE_PHASE_CODE): the energy level, - # for one, re-keys the transferred gate, not the assembled gate every - # later spine stage waits on (the re-review, E). + # take-up checks both phases declare (receipt_counts); the enum + # gate's nation partition (council_tax_constants, the region tier); + # and the engine the kernel is built with, through which each phase's + # enum gates resolve a domain no artifact supplies (the engine + # adapter's PolicyEngineUKEngine.enum_domain). What only one phase's + # gates compute through is that phase's own code, a node param + # (UK_SPINE_GATE_PHASE_CODE): the energy level, for one, re-keys the + # transferred gate and not the assembled gate every later spine stage + # waits on, since no stage before that gate hashes energy_pricing (the + # re-review, E). A phase-only module a stage before the assembled gate + # hashes still re-keys it by chaining + # (uk_spine_transferred_only_code_hash). return hashlib.sha256( canonical_json( { @@ -502,6 +525,7 @@ def implementation_hash(self) -> str: receipt_counts, council_tax_constants, geography_constants, + uk_engine_adapter, ), "country_resources": uk_solve_fingerprint(), } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index bde5a459e1..9a46a6bbb2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -1562,11 +1562,13 @@ def fixture_stage_plan_inputs( ) -> tuple[tuple[SourceStageSpec, ...], Mapping[str, object]]: """The H2 bundle as legacy StagePlan inputs: ordered specs, every transform. - The graph's CREATE node loads the captured root frame, so the kernel - mapping carries no ``frs_spine`` transform; the legacy oracle applies one - built from the same ``frs_raw`` tables, added here. The descriptor keys - stages by name, so the committed UK manifest supplies the order the - legacy plan runs them in. + The hermetic registry's transforms (``_fixture_implementations``) carry + no ``frs_spine`` transform, so its CREATE node, with no root transform + bound, loads the captured root frame. This mapping adds one built from + the same ``frs_raw`` tables: the legacy oracle applies it, and H2 parity + binds the graph's CREATE kernel to it, the root's production path. The + descriptor keys stages by name, so the committed UK manifest supplies the + order the legacy plan runs them in. """ from microcosm.build.country_spec import load_country_spec diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 95af501c00..8e6570d956 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -97,12 +97,12 @@ from .calibration_run import ( UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS, uk_local_gate_scope_exclusions, - uk_scoped_gate_manifest, ) from .cgt_projection import UK_CGT_PROJECTION_ARTIFACT_KEY from .content_identity import uk_frame_content_identity from .diagnostics import uk_target_geography_levels from .full_targets import CHRONICLE_SOURCE_CODEC, load_uk_national_target_inputs +from .gate_scopes import uk_scoped_gate_manifest from .graph_build import SPINE_PROVENANCE_TYPE from .graph_calibration import ( UKCalibrationNodes, @@ -1105,6 +1105,7 @@ def implementation_hash(self) -> str: diagnostics, donor_uprating, etb_services, + gate_scopes, graph_population, graph_targets, national_frame, @@ -1132,6 +1133,7 @@ def implementation_hash(self) -> str: gate_battery, battery_bindings, calibration_run, + gate_scopes, weighted_integrity, # The input-mass gate's source-household counts. receipt_counts, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 71fff0ef86..327721f37d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -826,8 +826,21 @@ def uk_terminal_only_code_hash() -> str: The source preflight runs the gate kernel's code and the dense solve keys on the preflight, so the code only the terminal phase computes through keys the calibrated node through its ``terminal_code`` parameter, not the - kernel's hash (microcosm#1306 review; microcosm#931 ruling 6): an edit to - it re-keys the calibrated gates and what follows them, never the solve. + kernel's hash (microcosm#1306 review; microcosm#931 ruling 6), and the + preflight's own key, and with it the solve's, carries none of it. An edit + to one of these modules re-keys the calibrated gates and what follows + them, and re-solves too wherever a node upstream of the solve hashes the + module, as a spine stage, a target or problem kernel, a population kernel + or the dense solve itself does for most of them. Nine never re-solve, on + a bound spine or in the in-graph build: ``high_value_home_gates``, + ``uk_runtime.diagnostics``, ``microcosm.diagnostics``' catalogs and + schema, ``calibrate.hierarchy_labels``, the target loss attribution, + ``input_mass``, ``cgt_projection`` and ``source_runtime``. + ``calibration_run`` and ``release_certification`` re-solve only on a bound + spine, whose kernel hashes both, and seven that upstream of the solve only + the spine's stage and gate nodes hash re-solve only in the in-graph build + (``donor_uprating``, ``etb_services``, ``etb_vat``, ``frs_take_up``, + ``hmrc_capital_gains``, ``take_up_contract``, ``uc_capital_coherence``). That is the terminal branch of the kernel's run, and what the terminal gates compute through beyond the preflight's: @@ -852,7 +865,8 @@ def uk_terminal_only_code_hash() -> str: (``graph_population``), the area-support summary (``local_rowwise``), the support floors, admin anchors and CGT projection it builds the gate context with (``support_floor``, ``etb_services``, - ``cgt_projection``, ``donor_uprating``), the input-mass reference + ``cgt_projection``, ``donor_uprating``, and the runner's admin totals + and projection artifact, ``calibration_run``), the input-mass reference (``weighted_integrity``) and the fit-weight records it rehydrates (``release_certification``); - the terminal gates' evaluators: the input-mass parity @@ -879,6 +893,7 @@ def uk_terminal_only_code_hash() -> str: from .. import chronicle_epoch, input_mass from . import ( + calibration_run, cgt_projection, chronicle_feed, dfe_spring_term, @@ -926,6 +941,7 @@ def uk_terminal_only_code_hash() -> str: release_certification, weighted_integrity, input_mass, + calibration_run, etb_services, etb_vat, domestic_energy_targets, @@ -956,9 +972,14 @@ def uk_preflight_only_code_hash() -> str: (``microcosm.build.ledger_targets``). The preflight's node carries it as its ``preflight_code`` parameter, so - an edit to it re-keys the preflight and the solve after it, and the - calibrated gates only through them (microcosm#931 ruling 6). Hashed once - per process, as the calibrated gates' own code is. + the calibrated gates' own key leaves it out (microcosm#931 ruling 6). An + edit to it re-keys the preflight, the solve after it and the calibrated + gates through them, and every other node that hashes it: the full build's + target compilation, selection, measures and problem, the national line's + targets, problem and gates, and in the in-graph build the + ``etb_services`` and ``lcfs_consumption`` stages and the transferred + spine gate, whose own code it is too. Hashed once per process, as the + calibrated gates' own code is. """ from .. import ledger_targets as shared_ledger_targets @@ -1019,10 +1040,14 @@ def implementation_hash(self) -> str: # encodes; the battery, the gate registry and the shared gates' # evaluators; and what both phases' gates compute through, among it # the release input coverage and the frame surface it reads, the - # target surfaces and their expected sets, and the register loaders - # (terminal_gates). What only one phase computes through is that - # phase's own code (UK_FULL_GATE_PHASE_CODE), so the preflight, and - # the dense solve keyed on it, re-key on none of the terminal phase's. + # target surfaces and their expected sets, the register loaders + # (terminal_gates), and the spine and local gate scopes (gate_scopes, + # not the runner that re-exports them). What only one phase computes + # through is that phase's own code (UK_FULL_GATE_PHASE_CODE), so the + # preflight's own key, and with it the dense solve's, carries none of + # the terminal phase's; an edit to one of those still re-solves + # wherever a node upstream of the solve hashes it + # (uk_terminal_only_code_hash). import microcosm.calibrate.geography_constants as geography_constants import microcosm.calibrate.hierarchy as calibrate_hierarchy import microcosm.calibrate.registry as calibrate_registry @@ -1034,9 +1059,9 @@ def implementation_hash(self) -> str: from .. import gate_battery, gates, stage_evidence from . import ( battery_bindings, - calibration_run, council_tax_constants, full_gates, + gate_scopes, geography_ladder, graph_evidence, graph_targets, @@ -1063,7 +1088,7 @@ def implementation_hash(self) -> str: stage_evidence, graph_evidence, graph_targets, - calibration_run, + gate_scopes, release_input_coverage, national_frame, uk_engine_adapter, @@ -2022,8 +2047,8 @@ def rowwise_candidate_manifest_from_graph( from microcosm.calibrate.artifacts import decode_problem from .battery_bindings import uk_area_support_below_floor - from .calibration_run import UK_LOCAL_GATE_SCOPE from .diagnostics import uk_fit_by_family, uk_support_limited_misses + from .gate_scopes import UK_LOCAL_GATE_SCOPE from .graph_targets import registry_from_payload from .national_sampling import UK_SAMPLE_RUNG_TOKENS from .rowwise_cli import ( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 58400a597d..8cb38dbf6f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -871,7 +871,10 @@ def test_the_engine_stages_bind_what_the_injected_engine_computes_through() -> N is rooted at that class (``_ENGINE_CLASS``): it follows the methods the stage's code names on it, by attribute or through ``getattr``, with the constructor and every method they call on ``self``, at the first-party - scope with the stage rule's tables (``_engine_walk``). ``_stage_walks`` + scope with the stage rule's tables (``_engine_walk``), and the methods + its own reach names on any receiver outside the engine's classes, as a + helper handed the engine might call, to a fixed point + (``_engine_reach``). ``_stage_walks`` adds it to the walk from the transform, so the rule's tests hold what it reaches like any other module the stage computes through. Every engine stage names ``materialize``, the RulesEngine protocol's call. Together the @@ -926,7 +929,10 @@ def test_the_engine_root_follows_the_methods_a_stage_calls( """Control for the second root: the methods a stage names on an engine, by attribute on any receiver or through ``getattr``, are followed, with the constructor and the methods they call on ``self``; a method no stage - names is not, nor what only it computes through.""" + names is not, nor what only it computes through. A method a helper the + engine hands itself to calls on that argument is followed too, to a fixed + point (``_engine_reach``), which one pass from the stage's methods + misses.""" import importlib import uuid @@ -963,21 +969,46 @@ def test_the_engine_root_follows_the_methods_a_stage_calls( " reader = getattr(model, 'parameters', None)\n" " return model.materialize(frame, ['x'], 2024), reader\n" ), + "delegating": ( + "from .engine import Engine\n\n" + "def _vocabulary_of(engine):\n" + " return engine.concept_mapping()\n\n" + "class Delegating(Engine):\n" + " def materialize(self, frame, names, period):\n" + " return _vocabulary_of(self)\n" + ), } for module, source in sources.items(): (package / f"{module}.py").write_text(source) prefix = f"microcosm.{scratch}." + + def short(reach): + return {module.removeprefix(prefix): names for module, names in reach.items()} + engine_class = importlib.import_module(f"{prefix}engine").Engine walk = _helper_walk((f"{prefix}stage", "Transform"), scope=prefix) methods = _engine_methods(engine_class, walk) assert methods == {"materialize", "parameters"} reach = _engine_walk(engine_class, methods, scope=prefix).reach - assert {module.removeprefix(prefix): names for module, names in reach.items()} == { + assert short(reach) == { "engine": {"Engine"}, "count": {"count"}, "tables": {"tables"}, "tree": {"tree"}, } + assert _engine_reach(engine_class, walk, scope=prefix).reach == reach + delegating = importlib.import_module(f"{prefix}delegating").Delegating + once = _engine_walk( + delegating, _engine_methods(delegating, walk), scope=prefix + ).reach + assert "vocabulary" not in short(once) + assert short(_engine_reach(delegating, walk, scope=prefix).reach) == { + "delegating": {"Delegating", "_vocabulary_of"}, + "engine": {"Engine"}, + "count": {"count"}, + "tree": {"tree"}, + "vocabulary": {"vocabulary"}, + } def test_the_engine_call_scan_sees_any_receiver(tmp_path) -> None: @@ -1347,10 +1378,10 @@ def changed_module_bytes(path): "maps each UK gate to its evaluator. The bound spine builds a battery " "run through it only for the checkpoint's gate digests, which read the " "manifest, and the run reads the registry only to refuse a parameter " - "its binding does not declare. The kernels that evaluate gates through " - "it are held past it (microcosm#931 ruling 6): each gate battery " - "phase to what the gates it declares compute through " - "(_GATE_BATTERY_KERNELS), and the national gates to its whole reach " + "its binding does not declare. The rule follows the kernels that " + "evaluate gates through it past it (microcosm#931 ruling 6): each gate " + "battery phase into what the gates it declares compute through " + "(_GATE_BATTERY_KERNELS), and the national gates into its whole reach " "(_WHOLE_REGISTRY_KERNELS)" ), } @@ -1362,9 +1393,14 @@ def changed_module_bytes(path): #: evaluates the gates its manifest declares for it. The rule holds each phase #: to what it computes through (``_battery_phase_walk``): its key, that hash #: with the phase's own code, which its node carries as ``_code``, -#: covers it and carries nothing only another phase computes through, so an -#: edit only one phase computes through re-keys that phase alone -#: (microcosm#931 ruling 6; test_each_gate_battery_phase_keys_on_its_own_reach). +#: covers it and carries nothing only another phase computes through, so no +#: other phase's own key re-keys on an edit only one phase computes through. +#: Key chaining still re-keys another phase where a node upstream of it hashes +#: the edited module: a stage before the assembled gate, for a module only +#: the transferred phase computes through, or a spine, target, population or +#: solve kernel before the preflight and the solve, for one only the terminal +#: phase does (microcosm#931 ruling 6; +#: test_each_gate_battery_phase_keys_on_its_own_reach). _GATE_BATTERY_KERNELS: dict[str, str] = { "uk.spine-gates@1": "spine", "uk.full-gates.preflight@1": "full", @@ -1401,7 +1437,9 @@ def changed_module_bytes(path): "domain its one declared column names, so the CGT asset type domain, " "which only a transferred-phase gate checks, is the transferred " "phase's alone, and the ONS household type domain the assembled " - "phase's; the rules engine is the kernel's own, no first-party code" + "phase's. The rules engine entry names no first-party definition: it " + "is the engine the kernel is built with, whose methods a phase's gates " + "call the walk follows from its class (_BATTERY_ENGINES)" ), } #: Artifacts a gate's evaluator reads beyond those its binding declares @@ -1917,13 +1955,17 @@ def _helper_reach( _ENGINE_CLASS = ("microcosm.frame.adapters.policyengine_uk", "PolicyEngineUKEngine") -def _engine_class() -> type: +def _class_named(ref: tuple[str, str]) -> type: import importlib - module, name = _ENGINE_CLASS + module, name = ref return getattr(importlib.import_module(module), name) +def _engine_class() -> type: + return _class_named(_ENGINE_CLASS) + + def _class_attributes(cls: type) -> dict[str, list[tuple[type, object]]]: """Each attribute ``cls`` defines, with the class and statement that define it, as ``_kernel_body`` chooses them by method resolution order.""" @@ -1961,13 +2003,22 @@ def _engine_methods(engine_class: type, walk: _Walk) -> frozenset[str]: return frozenset(found & set(_class_attributes(engine_class))) -def _engine_walk( +def _engine_roots(engine_class: type) -> frozenset[tuple[str, str]]: + """The classes an engine of ``engine_class`` comes from, the roots of its + walk. Their definitions name every method, so the engine methods a walk + names are read outside them (``_engine_reach``).""" + + return frozenset( + (cls.__module__, cls.__name__) for cls in _kernel_classes(engine_class) + ) + + +def _engine_seeds( engine_class: type, methods: frozenset[str], scope: str = _HELPER_SCOPE -) -> _Walk: - """What an engine of ``engine_class`` computes through for a stage that - calls ``methods`` on it: the stage rule's walk, at ``scope`` with the - same tables, from those methods, the constructor the builds call, and - every method they call on ``self``, to a fixed point.""" +) -> list[tuple[tuple[str, str], tuple[str, str]]]: + """What ``methods`` of ``engine_class``, the constructor the builds call + and every method they call on ``self`` name, each with the method naming + it: the seeds of the engine walk (``_engine_walk``).""" import ast @@ -1987,20 +2038,65 @@ def _engine_walk( and sub.value.id in {"self", "cls"} and sub.attr in body ) - seeds = [ + return [ (found, (owner.__module__, f"{owner.__name__}.{name}")) for name in sorted(chosen) for owner, statement in body[name] for found in _named(owner.__module__, statement, scope) ] + + +def _engine_walk( + engine_class: type, methods: frozenset[str], scope: str = _HELPER_SCOPE +) -> _Walk: + """What an engine of ``engine_class`` computes through for a stage that + calls ``methods`` on it: the stage rule's walk, at ``scope`` with the + same tables, from those methods, the constructor the builds call, and + every method they call on ``self``, to a fixed point.""" + walk, _, _ = _walk( - {(cls.__module__, cls.__name__) for cls in _kernel_classes(engine_class)}, - seeds, + set(_engine_roots(engine_class)), + _engine_seeds(engine_class, methods, scope), scope, ) return walk +def _engine_methods_beyond(engine_class: type, walk: _Walk) -> frozenset[str]: + """The methods of ``engine_class`` that ``walk`` names outside the + engine's own classes (``_engine_methods``, ``_engine_roots``).""" + + roots = _engine_roots(engine_class) + return _engine_methods( + engine_class, + _Walk( + { + module: frozenset(name for name in names if (module, name) not in roots) + for module, names in walk.reach.items() + }, + walk.stops, + walk.core, + ), + ) + + +def _engine_reach(engine_class: type, walk: _Walk, scope: str = _HELPER_SCOPE) -> _Walk: + """What an engine of ``engine_class`` computes through for code whose + walk is ``walk``: the engine walk (``_engine_walk``) from the methods + ``walk`` names, and from those the engine walk's own reach names on any + receiver outside the engine's classes, to a fixed point. So a helper the + engine hands itself to, which calls another of its methods on that + argument, is followed into that method too.""" + + methods = _engine_methods(engine_class, walk) + while True: + reached = _engine_walk(engine_class, methods, scope) + more = methods | _engine_methods_beyond(engine_class, reached) + if more == methods: + return reached + methods = more + + def _merged(first: _Walk, second: _Walk) -> _Walk: reach = dict(first.reach) for module, names in second.reach.items(): @@ -2065,16 +2161,15 @@ def _transform_walks() -> MappingProxyType: @functools.cache def _engine_root_walks() -> MappingProxyType: - """Each engine stage's second walk (``_engine_walk``): from the methods - of the injected engine its transform's walk names.""" + """Each engine stage's second walk (``_engine_reach``): from the methods + of the injected engine its transform's walk names, and those the engine + walk's own reach names, to a fixed point.""" engine_class = _engine_class() walks = _transform_walks() return MappingProxyType( { - stage: _engine_walk( - engine_class, _engine_methods(engine_class, walks[stage]) - ) + stage: _engine_reach(engine_class, walks[stage]) for stage in sorted(_engine_stages()) } ) @@ -2871,7 +2966,12 @@ def _artifacts_read(entry) -> frozenset[str]: return frozenset(read) -def _battery_phase_walk(kernel_class: type, gates, phase: str) -> _KernelWalk: +def _battery_phase_walk( + kernel_class: type, + gates, + phase: str, + engine: tuple[tuple[str, str], frozenset[str]] | None = None, +) -> _KernelWalk: """What one phase of a gate battery computes through (microcosm#931 ruling 6): the kernel rule's walk from what the kernel runs, each ``if phase == ...`` taking the branch ``phase`` takes (``_phase_branch``), and @@ -2882,8 +2982,12 @@ def _battery_phase_walk(kernel_class: type, gates, phase: str) -> _KernelWalk: dispatcher is followed only into the branches of the checks the phase declares (``_STAGE_HEALTH_DISPATCH``), and an evidence-artifact table the walk meets only into the entries the phase's gates read - (``_EVIDENCE_ARTIFACT_TABLES``, ``_artifacts_read``). ``met`` holds the - listed kernel stops it meets.""" + (``_EVIDENCE_ARTIFACT_TABLES``, ``_artifacts_read``). ``engine`` is the + class the battery's kernel is built with and the artifacts that hand it + to the gates (``_BATTERY_ENGINES``): where the phase's gates read one, + the engine's walk from the methods the phase's reach names joins the + phase's, to a fixed point, as for an engine stage (``_engine_reach``). + ``met`` holds the listed kernel stops it meets.""" entries = [ entry @@ -2930,12 +3034,27 @@ def _battery_phase_walk(kernel_class: type, gates, phase: str) -> _KernelWalk: _STAGE_HEALTH_DISPATCH: "the stage-health dispatch", **_EVIDENCE_ARTIFACT_TABLES, } + # A phase whose gates read the engine the kernel hands them computes + # through the methods they call on it. + handed = ( + _class_named(engine[0]) if engine is not None and read & engine[1] else None + ) + methods: frozenset[str] = frozenset() followed: set[tuple[str, str]] = set() while True: walk, met, core_names = _walk(roots, seeds, _HELPER_SCOPE, halts=halts) tables = (set(_EVIDENCE_ARTIFACT_TABLES) & set(met)) - followed - if not tables: + more = ( + methods | _engine_methods_beyond(handed, walk) + if handed is not None + else methods + ) + if not tables and more == methods: break + if more != methods: + roots |= _engine_roots(handed) + seeds.extend(_engine_seeds(handed, more)) + methods = more for table in sorted(tables): roots.add(table) seeds.extend( @@ -2954,6 +3073,29 @@ def _battery_phase_walk(kernel_class: type, gates, phase: str) -> _KernelWalk: ) +#: The engine each gate battery's kernel is built with, and the evidence +#: artifacts through which its run hands that engine to the gates, by +#: battery. The spine gates hand the rules engine the spine build injects +#: (spine_build._rules_engine) as ``rules_engine`` (uk_spine_gate_artifacts). +#: The full build's preflight and calibrated gates hand the coverage engine +#: the driver builds them with as ``coverage_engine``, and the gate context +#: hands it on as ``rules_engine`` (full_gates.build_full_gate_context). A +#: phase whose gates read one computes through the methods they call on the +#: engine, which the walk follows from its class (``_battery_phase_walk``), +#: as the engine stages' walks follow the engine they call (microcosm#931 +#: ruling 5; test_each_gate_battery_walks_the_engine_its_build_hands_it). +_BATTERY_ENGINES: dict[str, tuple[tuple[str, str], frozenset[str]]] = { + "spine": (_ENGINE_CLASS, frozenset({"rules_engine"})), + "full": ( + ( + "microcosm.build.uk_runtime.release_input_coverage", + "PolicyEngineUKCoverageEngine", + ), + frozenset({"coverage_engine", "rules_engine"}), + ), +} + + class _Battery(NamedTuple): """A gate battery: the manifest its kernels evaluate, the kernel each phase runs under, and the table of each phase's own code.""" @@ -3037,7 +3179,12 @@ class with several kernels (the spine's stage kernels) is walked once. shared = _hashed_modules(kernel) rule[ref] = { f"{ref} {phase}": _KeyedWalk( - _battery_phase_walk(kernel_class, battery.gates, phase), + _battery_phase_walk( + kernel_class, + battery.gates, + phase, + _BATTERY_ENGINES[_GATE_BATTERY_KERNELS[ref]], + ), shared | _phase_code_modules(battery, phase), ) for phase, runner in battery.phases.items() @@ -3225,12 +3372,14 @@ def test_each_gate_battery_phase_keys_on_its_own_reach() -> None: only some of them do. Each phase's own code, which its node carries as ``_code``, holds exactly the rest of what that phase computes through, and a phase that computes through nothing of its own carries - none. So an edit only the transferred spine phase computes through - re-keys neither the assembled gate nor the spine stages that wait on it, - and one only the terminal phase computes through re-keys neither the - preflight nor the dense solve keyed on it. What a phase computes through - is its walk (``_battery_phase_walk``), which the kernel rule's test holds - each phase's key to cover.""" + none. So the assembled gate's own key, which the spine stages after it + chain on, carries nothing only the transferred phase computes through, + and the preflight's, which the dense solve chains on, nothing only the + terminal phase does. An edit to such a module still re-keys the sibling + phase where a node upstream of it hashes the module (graph_evidence's and + graph_terminal's phase-code docstrings name where). What a phase computes + through is its walk (``_battery_phase_walk``), which the kernel rule's + test holds each phase's key to cover.""" from microcosm.build.uk_runtime.graph_evidence import ( UK_SPINE_GATE_PHASE_CODE, add_uk_spine_gate_nodes, @@ -3277,6 +3426,94 @@ def test_each_gate_battery_phase_keys_on_its_own_reach() -> None: assert params[f"{phase}_code"] == code(), phase +def test_each_gate_battery_walks_the_engine_its_build_hands_it() -> None: + """The engine a battery's phase walk follows (``_BATTERY_ENGINES``) is + the one its build constructs the battery's kernel with, and the kernel + hands it to the gates as one of the battery's engine artifacts + (microcosm#931 review): + + - the spine build hands the spine gates the engine ``_rules_engine`` + builds, the class the engine stages' root follows + (test_the_engine_root_is_the_engine_the_builds_inject), and the gates' + artifact table hands it on as ``rules_engine``; + - the full build's driver hands the preflight and calibrated gates a new + coverage engine, and their source evidence hands it on as + ``coverage_engine``; + - each spine phase's enum gates read the rules engine, so each phase's + walk reaches the engine's class, through which they resolve a domain no + artifact supplies, and the spine gates' shared hash binds its module.""" + import ast + + from microcosm.build.uk_runtime import ( + full_build_cli, + graph_evidence, + graph_terminal, + spine_build, + ) + + for module, register, keyword, battery in ( + (spine_build, "register_spine_gate_kernel", "engine", "spine"), + (full_build_cli, "register_uk_full_gate_kernels", "coverage_engine", "full"), + ): + tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) + handed = [] + for function in ast.walk(tree): + if not isinstance(function, ast.FunctionDef): + continue + bound = { + target.id: sub.value + for sub in ast.walk(function) + if isinstance(sub, ast.Assign) + for target in sub.targets + if isinstance(target, ast.Name) + } + for call in ast.walk(function): + if not ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Name) + and call.func.id == register + ): + continue + (value,) = [item.value for item in call.keywords if item.arg == keyword] + if isinstance(value, ast.Name): + value = bound[value.id] + assert isinstance(value, ast.Call), (module.__name__, register) + assert isinstance(value.func, ast.Name), (module.__name__, register) + handed.append(_resolve(module.__name__, value.func.id, _HELPER_SCOPE)) + engine = _BATTERY_ENGINES[battery][0] + expected = ( + (spine_build.__name__, "_rules_engine") if battery == "spine" else engine + ) + assert handed and set(handed) == {expected}, (battery, handed) + + for module, function, artifact in ( + (graph_evidence.__name__, "uk_spine_gate_artifacts", "rules_engine"), + (graph_terminal.__name__, "_source_gate_evidence", "coverage_engine"), + ): + node = _module_definitions(module, _HELPER_SCOPE).definitions[function] + (table,) = [sub.value for sub in ast.walk(node) if isinstance(sub, ast.Return)] + assert isinstance(table, ast.Dict), function + values = { + key.value: value + for key, value in zip(table.keys, table.values, strict=True) + if isinstance(key, ast.Constant) + } + assert ast.unparse(values[artifact]) == "engine", function + assert ( + artifact + in _BATTERY_ENGINES[ + "spine" if function == "uk_spine_gate_artifacts" else "full" + ][1] + ) + + rule = _kernel_rule_walks() + engine_module, engine_name = _ENGINE_CLASS + for phase in _gate_batteries()["spine"].phases: + walk = rule["uk.spine-gates@1"][f"uk.spine-gates@1 {phase}"].walk + assert engine_name in walk.reach.get(engine_module, ()), phase + assert engine_module in _hashed_modules(_uk_graph_kernels()["uk.spine-gates@1"]) + + def test_the_battery_model_reads_the_declared_dispatch() -> None: """The per-phase walk models how a battery reaches each evaluator (``_battery_phase_walk``), and each part of the model holds against the @@ -3553,8 +3790,8 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: # The gate registry: the bound spine derives its gate digests through it, # in _manifest_digests, which returns the battery run's digests alone. - # The gate batteries' runs evaluate through it, and each phase is held - # past it to the gates it declares; the national gates' run evaluates + # The gate batteries' runs evaluate through it, and each phase is followed + # past it into the gates it declares; the national gates' run evaluates # through it too, and they are walked past it to its whole reach. registry = _GATE_REGISTRY assert namers[registry] == { @@ -3666,27 +3903,39 @@ def test_no_spine_or_national_line_kernel_hashes_the_source_codecs() -> None: microcosm.graph.codecs (microcosm#931 ruling 8): no spine kernel hashes it, so an edit to it re-keys no spine node, nor any node downstream of the spine through it, and no kernel of the national line, whose root is - the bound spine checkpoint, hashes it either. The atomic geography - kernels, which read their sources' bytes through it, do.""" - from microcosm.build.uk_runtime import ( - graph_build, - graph_calibration, - graph_national, - ) + the bound spine checkpoint, hashes it either: each kernel the national + graph's nodes run, read from that graph, and the readback of its output. + The atomic geography kernels, which read their sources' bytes through it, + do.""" + from microcosm.build.uk_runtime import graph_national + from microcosm.build.uk_runtime.graph_build import bound_spine_graph + from microcosm.build.uk_runtime.national_doctrine import uk_doctrine_with_overrides from microcosm.graph import codecs + from test_support.microcosm_build import uk_calibration_run as seam spine = uk_registry(graph=uk_spine_graph()) for ref in spine.refs(): assert codecs.__name__ not in _hashed_modules(spine.get(ref)), ref - kernels = _uk_graph_kernels() - for ref in ( - graph_build.UKBoundSpineKernel.ref, - graph_national.UKNationalTargetKernel.ref, - graph_national.UKNationalProblemKernel.ref, - graph_calibration.UKDenseSolveKernel.ref, + doctrine, receipt = uk_doctrine_with_overrides(epochs=5) + national = graph_national.uk_national_graph( + graph_national.UKNationalBuildConfig( + calibration_year=2025, + time_period="2023", + source_year=2023, + doctrine=doctrine, + doctrine_overrides=receipt, + ), + spine=bound_spine_graph(seam._frame()), + spine_population="uk.full.spine_checkpoint", + review_date="2026-09-10", + ) + line = {node.kernel for node in national.graph.nodes} + assert { + "uk.full.bound_spine@1", graph_national.UKNationalGateKernel.ref, - graph_national.UKNationalReadbackKernel.ref, - ): + } <= line + kernels = _uk_graph_kernels() + for ref in sorted(line | {graph_national.UKNationalReadbackKernel.ref}): assert codecs.__name__ not in _hashed_modules(kernels[ref]), ref assert codecs.__name__ in _hashed_modules(kernels["geography.support_import@1"]) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 329ccba3d7..b0775c29a7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1334,22 +1334,22 @@ class TestSupportSplitRecomputeAcrossGeographicClones: The fold re-sums the copies, so neither the count check nor the family test recounts it. At any clone count a family of more than three members is held exactly where at least four of its members show their two clone - halves at equal weights and share one pair sum ``c``, whatever the others - carry: the stored ``n`` must be the count of a weight that divides by - ``n`` to ``c``. Where every member shares ``c`` but fewer show equal - halves, the weight may divide to within one float of ``c``, or two where - ``c`` sits that close to a power of two, a reach that at one clone can - take in the multiple of 60 itself, where a family split consistently into - one copy more goes unreported. Every other family, and whether each - household is a family at all, is held to the family weights' relative - tolerance of 1e-9, the counts at the folded weight times ``1 - 1e-9`` and - ``1 + 1e-9``, unless the hold decides. That bracket opens for a household - whose source-scaled weight ``K x w`` lies within a relative 1e-9 of a - multiple of 60 (at ``K = 1``, its own weight), where it cannot tell ``n`` - copies from ``n + 1``, or a family of two from a whole household: a - stored count moved on its own is left to the flags and the family - weights, and a household split consistently into one copy more goes - unreported. + halves at equal weights and every member that does shares one pair sum + ``c``, whatever the others carry: the stored ``n`` must be the count of a + weight that divides by ``n`` to ``c``. Where every member shares ``c`` but + fewer show equal halves, the weight may divide to within one float of + ``c``, or two where ``c`` sits that close to a power of two, a reach that + at one clone can take in the multiple of 60 itself, where a family split + consistently into one copy more goes unreported. Every other family, and + whether each household is a family at all, is held to the family weights' + relative tolerance of 1e-9, the counts at the folded weight times + ``1 - 1e-9`` and ``1 + 1e-9``, unless the hold decides. That bracket opens + for a household whose source-scaled weight ``K x w`` lies within a + relative 1e-9 of a multiple of 60 (at ``K = 1``, its own weight), where it + cannot tell ``n`` copies from ``n + 1``, or a family of two from a whole + household: a stored count moved on its own is left to the flags and the + family weights, and a household split consistently into one copy more + goes unreported. """ def test_at_one_clone_the_receipt_is_the_rule_before_the_clone_count( @@ -1645,6 +1645,47 @@ def test_at_one_clone_four_equal_halves_vouch_for_a_family_whose_others_differ( assert held_exactly == exact assert within == 0 + def test_at_one_clone_the_exact_tier_takes_its_weight_from_the_equal_halves( + self, + ) -> None: + # The correct stage splits a household a float above 900 into 16, + # every member at w / 16. One member's halves are re-split unevenly + # with its pair sum one float below that weight, so the least pair sum + # over the family's members lies a float below w / 16. The other 15 + # show equal halves sharing w / 16, so the hold takes the family + # exactly on their weight and accepts the stage's 16. Taking the least + # pair sum over every member instead would divide to the float below, + # whose weight 900 counts 15 copies, and report the correct stage. + scenario = { + "weights": [_ulps_above(900.0, 1), 9_000.0], + "wealth": [7e3, 1.0], + "incomes": [BAND_INCOMES[0], BAND_INCOMES[37_700]], + "top_cells": {0: {250_000: 1_000_000.0}}, + } + split, frame = _clone_first_split_then_cloned(1, scenario) + count = 16 + assert split.copies_created == count - 1 + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + members = np.r_[ + np.int64(1), + copy_id_table.mint([1] * (count - 1), "cgt_support_split", range(1, count)), + ] + clones = copy_id_table.mint(members, "cgt_incidence_clone", 1) + member, clone = members[-1], clones[-1] + shared = weights[ids == member][0] + weights[ids == clone][0] + below = float(np.nextafter(shared, 0.0)) + weights[ids == clone] = 16.0 + weights[ids == member] = below - 16.0 + assert weights[ids == member][0] + weights[ids == clone][0] == below + + _, problems, reaches = _hold_reaches( + _rebuild(frame, household_weights=weights), scenario + ) + + assert problems == {} + assert reaches == {0: 1} + @pytest.mark.parametrize("clone_count", [1, 3]) def test_a_family_of_three_or_fewer_is_left_to_the_bracket( self, monkeypatch, clone_count: int @@ -2142,12 +2183,15 @@ def test_the_hold_counts_every_weight_change_after_the_split(self) -> None: } assert boundaries == {"uk.identity@1"} - # Each call that corrects a frame's total, by the function or method - # making it: a call to spi_support's helpers, or to a function of the - # stage's module that wraps them, by bare name or as an attribute - # (spi_support._importance_weights_with_exact_total(...)). Every - # function, method and nested function counts its own calls, not - # those of the functions it nests. + # Each reference that can correct a frame's total, by the function or + # method making it: to spi_support's helpers, or to a function of any + # of the three stages' modules that wraps them (one named _exact... + # that refers to a helper or another wrapper), by bare name or as an + # attribute (spi_support._importance_weights_with_exact_total(...), + # cgt_support._exact_support_weights(...)), called or not, so an + # alias or a partial of one counts too. Every function, method and + # nested function counts its own references, not those of the + # functions it nests. helpers = {"_exact_total_correction", "_importance_weights_with_exact_total"} stages = ("cgt_support_split", *weighing) modules = {graph_kernels._stage_module(stage) for stage in stages} @@ -2167,8 +2211,9 @@ def collect(node, prefix: str, into: dict[str, ast.AST]) -> None: else: collect(child, prefix, into) - def called(function) -> list[str]: - """The names ``function`` calls outside the functions it nests.""" + def referred(function) -> list[str]: + """The names ``function`` refers to outside the functions it nests: + each bare name and each attribute read, a call's included.""" names: list[str] = [] pending = list(ast.iter_child_nodes(function)) @@ -2176,14 +2221,14 @@ def called(function) -> list[str]: node = pending.pop() if isinstance(node, nests): continue - if isinstance(node, ast.Call): - name = getattr(node.func, "id", getattr(node.func, "attr", None)) - if name is not None: - names.append(name) + if isinstance(node, ast.Name): + names.append(node.id) + elif isinstance(node, ast.Attribute): + names.append(node.attr) pending.extend(ast.iter_child_nodes(node)) return names - corrections: dict[tuple[str, str], int] = {} + every: dict[str, dict[str, ast.AST]] = {} for module in modules: tree = ast.parse(inspect.getsource(module)) functions: dict[str, ast.AST] = {} @@ -2191,18 +2236,29 @@ def called(function) -> list[str]: assert { node.name for node in ast.walk(tree) if isinstance(node, defined) } == {function.name for function in functions.values()} - wrappers = { + every[module.__name__] = functions + # The wrappers of all three modules, to a fixed point, so a wrapper + # one module defines counts where another module calls it. + wrappers: set[str] = set() + while True: + more = { function.name + for functions in every.values() for function in functions.values() if function.name.startswith("_exact") - and set(called(function)) & helpers + and set(referred(function)) & (helpers | wrappers) } + if more == wrappers: + break + wrappers = more + corrections: dict[tuple[str, str], int] = {} + for module_name, functions in every.items(): for name, function in functions.items(): if function.name in wrappers: continue - count = sum(item in helpers | wrappers for item in called(function)) + count = sum(item in helpers | wrappers for item in referred(function)) if count: - corrections[(module.__name__, name)] = count + corrections[(module_name, name)] = count assert corrections == { (cgt_support.__name__, "split_cgt_support_households"): 1, (cgt_structure.__name__, "clone_cgt_incidence"): 1, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index b02a6a6e68..c49144f0d0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -1,4 +1,4 @@ -"""Both UK target kernels hash every module that decides a target value. +"""The UK target and problem kernels hash every module that decides a target. Two checks pin the closure: @@ -6,12 +6,17 @@ first-party import of a module in ``UK_TARGET_COMPILE_MODULE_NAMES`` is listed, or excluded with a reason; - each kernel's: the derived binding rule in test_uk_graph.py holds what each - target and problem kernel's hash covers to what its code computes through + of the six target and problem kernels' hashes covers to what its code + computes through (``test_every_uk_graph_kernel_hashes_each_helper_it_computes_through``). It replaced microcosm#1497's per-kernel import record for these kernels - (microcosm#931 ruling 6), so this file keeps only what that rule does not - cover: the modules a kernel's file imports only to build its hash - (``HASH_INPUTS``) stay out of the hashes. + (microcosm#931 ruling 6). + +Beside pins of what the kernels hash, this file keeps what that rule does +not cover. Each module a kernel's file imports only to build a hash +(``HASH_INPUTS``) is still imported where listed, and a kernel hashes one +only where another module it hashes imports it too. No kernel hashes the +compile list or ``solve_scope``. """ from __future__ import annotations @@ -292,8 +297,9 @@ def test_an_edit_to_calibration_run_leaves_the_kernels_keys( ): """The target kernels run only the two functions now in ledger_surface. An edit elsewhere in calibration_run, the national calibration runner, - re-runs no target compilation and, in builds that construct the spine in - the graph, no solve (microcosm#1492).""" + re-runs no target compilation (microcosm#1492) and, in builds that + construct the spine in the graph, no solve: the preflight the solve keys + on leaves the runner out too (the test below).""" from microcosm.build.uk_runtime import calibration_run before = kernel.implementation_hash() @@ -304,6 +310,44 @@ def test_an_edit_to_calibration_run_leaves_the_kernels_keys( assert kernel.implementation_hash() == before +def test_an_edit_to_calibration_run_leaves_the_preflight_and_the_solve( + tmp_path, monkeypatch +): + """The full gates' preflight computes through the spine and local gate + scopes and the scoped gate manifest, which moved unchanged from + calibration_run into gate_scopes, so its key, and with it the dense + solve's, hashes the small module and not the runner (microcosm#931), as + the target kernels hash ledger_surface. Neither full gate kernel's own + hash nor the preflight's own code moves on an edit to the runner. The + calibrated gates' own code binds it, for the admin totals and the CGT + projection artifact their gate context is built with, so of the two + phases only the calibrated gates re-key.""" + from microcosm.build.uk_runtime import calibration_run, gate_scopes, graph_terminal + + kernels = [ + kind(coverage_engine=object(), engine_identity="test") + for kind in ( + graph_terminal.UKFullPreflightGateKernel, + graph_terminal.UKFullGateKernel, + ) + ] + preflight_code = graph_terminal.uk_preflight_only_code_hash.__wrapped__ + terminal_code = graph_terminal.uk_terminal_only_code_hash.__wrapped__ + for kernel in kernels: + hashed = hashed_module_names(kernel, graph_terminal) + assert gate_scopes.__name__ in hashed, kernel.ref + assert calibration_run.__name__ not in hashed, kernel.ref + before = [kernel.implementation_hash() for kernel in kernels] + codes = (preflight_code(), terminal_code()) + source = Path(calibration_run.__file__) + edited = tmp_path / source.name + edited.write_bytes(source.read_bytes() + b"\n# an edited line\n") + monkeypatch.setattr(calibration_run, "__file__", str(edited)) + assert [kernel.implementation_hash() for kernel in kernels] == before + assert preflight_code() == codes[0] + assert terminal_code() != codes[1] + + def test_full_problem_loss_weights_hash_what_target_weights_reads(monkeypatch): """target_weights takes UK_WIDE_GEOGRAPHY_IDS from geography_constants and the loss scales from calibrate.solve for the weights the problem diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index bbb7dbb8e3..f8308aa40f 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -857,8 +857,9 @@ def _e8_support_split( is recounted from the fold. At any ``K`` a family of more than three members is held exactly where it can be (``_support_count_held_exactly``). Where at least four of its members show their two clone halves at equal - weights and share one pair sum ``c``, whatever the others carry, the - stored ``n`` must be the count of a weight ``w`` with ``fl(w / n) == c``. + weights and every member that does shares one pair sum ``c``, whatever + the others carry, the stored ``n`` must be the count of a weight ``w`` + with ``fl(w / n) == c``. Where every member shares ``c`` but fewer show equal halves, ``fl(w / n)`` may lie within one float of ``c``, or two where ``c`` sits that close to a power of two, the most the anchor and the arms move an @@ -2076,8 +2077,9 @@ def plans_equal(left: pd.Series, right: pd.Series) -> bool: "1e-9; support-split copy counts, at any K: for a family of more " "than three members, exactly the count of a weight w with " "fl(w / n) == c where at least four members show their two clone " - "halves at equal weights and share the pair sum c, whatever the " - "others carry, or, where every member's pair sum is c but fewer " + "halves at equal weights and every member that does shares the " + "pair sum c, whatever the others carry, or, where every member's " + "pair sum is c but fewer " "show equal halves, with fl(w / n) within one float of c, or two " "where c sits within two floats of a power of two; either run " "can reach across the count's step from the floats next to it " From 63f825d03476c06ffc0404da87c943a5a3f87604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:54:37 +0000 Subject: [PATCH 040/108] Key the CGT redraw's walk offsets and plan jitters, and BADR's band offsets, on their labels (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María's ruling 6 of 7 Nov: key cgt_imputation's shared stream, and BADR's band offsets on the band bounds. Stage 44 built one Generator from its seed (552) and the build period and handed it to every Table 3 cell walk, pooled walk and plan in run order, so the n-th uniform belonged to whichever walk or plan ran n-th: a cell that gained or lost its gainers, or skipped, moved the uniform of every walk and plan after it. Stage 47 drew one offset per band by position, so a band added to the facts moved every band after it. - Shared scalar: stochastic_assignment.stable_identity_uniform(key, *, seed, salt) is stable_identity_uniforms' draw for one label, and refuses a sequence. The random() adapter, _KeyedOffset, moves from isa_subscriptions into cgt_imputation beside _walk_bands; the ISA subscriptions take it and the scalar from there. - Stage 44: a cell walk's offset is keyed cell::: and a pooled walk's pool:, salted hmrc_cgt_gains_spine:offset:; a plan's jitter plan::, salted hmrc_cgt_gains_spine:jitter:. Bands and age groups are named by their lower bounds, region groups by their labels. The seed stays 552 and the period moves into the salts. Each walk and plan reads one uniform. _walk_bands and _draw_plan_amounts keep their signatures; their rng annotation names the adapter too. - Stage 47: a band's offset is keyed | (open for the top band), salted assign_badr_qualifying_gains:offset:, seed 555 (_badr_band_offset). - Declarations: stage 44's seed_mixing and stage 47's realization texts say this, and the sub-AEA mapping text no longer names a walk and plan draw order, which no longer exists. The code dictionaries, sources.yaml and source_stages.json agree; the docstrings and comments follow. - Bindings: the derived rule now finds stochastic_assignment in the reach of stages 44 and 47, so both bind it. test_uk_graph's invalidation test takes regional_property_uprating and private_school_attendance as controls in their place. - property_components' receipts-walk offset is unchanged (ruling 6). No nesting is promised. The offsets and jitters are the same at every K and whichever other cells walk, but the walks stay pooled: the rounding carry within an income band, pass 2's walk over the band's unassigned gainers, plans pooled across cells, and each BADR band's walk over its pool. What moves (synthetic counts only): - Stage 44: every walk offset and plan jitter, so band membership near each walk boundary and the within-band quantiles. On three synthetic frames, 9 of 1,002, 9 of 3,000 and 126 of 1,512 gainers changed gain band and 338, 2,920 and 1,300 changed amount; no non-gainer moved. - Stage 47: every band's offset, so each band's claimants. On three synthetic frames of equal weights no claimant row of the old walk claims in the new one, each frame's claimant count moved by one or two, and the main asset types moved with the claims. - ISA: unchanged. On three synthetic frames all five output columns are bit for bit the parent's (4,881 person rows). - Node keys: all 79 spine nodes' implementation hashes move, since graph_kernels and stochastic_assignment changed and the root binds stochastic_assignment, so every node re-keys, as does every node keyed on the spec or the coverage manifest. Values move from stage 44 on; isa_subscriptions re-keys through its cgt_imputation binding and its own module with no change in its output. - Regenerated: source_stages.json; the coverage manifest's 25 source digests; in the H2 fixture, the three texts in fixture.json, the two stages' contract hashes in uk_spine.json, and 22 values of the synthetic cgt_asset_type_facts.json, which the tool sizes to the frame after the redraw. No gates.json note goes stale. Tests (engine-free, synthetic or code facts): - New: the scalar is the array helper's draw bit for bit and refuses a sequence. Each walk's and plan's uniform, recorded under a label taken from the call's own members, is the keyed scalar of that label, salted with the period. It is unchanged when an earlier cell gains members or skips, and the one-cell higher income bands keep their members and amounts. The redraw is invariant to a row permutation. A band inserted into Table 4.1 leaves the other bands' offsets and claims as they were. ISA's offsets and jitters are the uniforms they were. - 16 mutations of a scratch copy are each caught, and the unmodified copy passes. The shared Generator as before, and offsets keyed on the walk's position, fail the invariance tests (under the shared stream 54 of the higher bands' 60 plan amounts move); mis-keyed cells, pools or plans, salts without the period and one salt for offsets and jitters fail the label tests; BADR's positional offsets as before, a position key and a lower-bound key fail the insertion test; and the ISA, scalar and binding mutations fail theirs. - The seven touched test files pass (230 tests), as do the coverage manifest, declared-seed and replay-artifact tests (31). Ruff is clean on the 10 edited Python files, and ci_test_plan verify is ok. source_stages.json and the coverage manifest are current, the H2 fixture regenerates and passes its check, and H2 parity passes through the engine slot. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-cgt-keyed-offsets.changed.md | 12 + .../microcosm/build/stochastic_assignment.py | 18 ++ .../uk/release_input_coverage_manifest.json | 50 ++-- .../src/microcosm/build/uk/source_stages.json | 6 +- .../src/microcosm/build/uk/spec/sources.yaml | 6 +- .../build/uk_runtime/cgt_asset_type.py | 49 +++- .../build/uk_runtime/cgt_imputation.py | 73 +++-- .../build/uk_runtime/graph_kernels.py | 7 +- .../build/uk_runtime/isa_subscriptions.py | 26 +- .../shared/test_stochastic_assignment.py | 25 ++ .../engine_free/uk/test_uk_cgt_asset_type.py | 72 +++++ .../engine_free/uk/test_uk_cgt_imputation.py | 262 ++++++++++++++++++ .../tests/engine_free/uk/test_uk_graph.py | 7 +- .../uk/test_uk_isa_subscriptions.py | 32 +++ .../sources/cgt_asset_type_facts.json | 44 +-- .../parity/uk_spine/sources/fixture.json | 6 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- 17 files changed, 590 insertions(+), 107 deletions(-) create mode 100644 changelog.d/931-cgt-keyed-offsets.changed.md diff --git a/changelog.d/931-cgt-keyed-offsets.changed.md b/changelog.d/931-cgt-keyed-offsets.changed.md new file mode 100644 index 0000000000..b1825b9c51 --- /dev/null +++ b/changelog.d/931-cgt-keyed-offsets.changed.md @@ -0,0 +1,12 @@ +The CGT gains redraw (`hmrc_cgt_gains_spine`) and the BADR claims in the asset-type stage (`hmrc_cgt_asset_type_spine`) key each systematic walk's offset, and each plan's quantile jitter, on its own label instead of taking the next draw of one Generator (#931; María's ruling 6 of 7 Nov). The redraw built one Generator from its seed (552) and the build period and handed it to every Table 3 cell walk, every income band's pooled walk and every (gain band, income band) plan in run order, so the n-th uniform belonged to whichever walk or plan ran n-th: a cell that gained or lost its gainers, or skipped on its carried target, moved the uniform of every walk and plan after it. Each is now an identity-keyed uniform under the declared seed, read once: + +- a cell walk is keyed `cell:::` and an income band's pooled walk `pool:`, both salted `hmrc_cgt_gains_spine:offset:`; +- a plan is keyed `plan::`, salted `hmrc_cgt_gains_spine:jitter:`; +- a BADR band is keyed on its bounds, `|` (`open` for the top band), salted `assign_badr_qualifying_gains:offset:`, under the BADR seed (555). The band offsets never depended on rows, but each was its band's position in one draw of every band's offset, so a facts revision that added a band moved the offset of every band after it. + +Bands and age groups are named by their lower bounds, region groups by their labels. The period moved from the Generator's seed into each salt, so the declarations still read "seed combined with the build period". + +- **Shared scalar.** `microcosm.build.stochastic_assignment.stable_identity_uniform(key, *, seed, salt)` returns the draw `stable_identity_uniforms([key], ...)` gives for one label, and refuses a sequence. The duck-typed `random()` adapter the walks read (`_KeyedOffset`) moved from `isa_subscriptions` into `cgt_imputation`, beside `_walk_bands`; the ISA subscriptions take it and the shared scalar from there, and their walk offsets and band jitters are the uniforms they were, bit for bit. `property_components`' one receipts-walk offset stays as it is (ruling 6). +- **Declarations.** The redraw's `seed_mixing` and the BADR `realization` texts say the above, and the sub-AEA remainder's `mapping` text no longer names a walk and plan draw order, since there is none. Each is identical in the code dictionary, `sources.yaml` and `source_stages.json`. `hmrc_cgt_gains_spine` and `hmrc_cgt_asset_type_spine` now compute through `stochastic_assignment`, so both bind it, as the derived binding rule requires. +- **No nesting promised.** The offsets and jitters are the same at every geographic clone count, and whichever other cells walk or skip. The walks themselves stay pooled: the rounding carry runs through an income band's cells, the pooled walk takes the band's net shortfall over its unassigned gainers, each plan stratifies members pooled from every cell, and each BADR band walks its whole pool. A change in one cell can still move the assignments and amounts of the other cells in its income band, and a plan's amounts whenever its members change. +- **What moves.** Every Table 3 walk's offset and every plan's jitter changes, so band membership near each walk boundary and the within-band quantiles move. On three synthetic frames, 9 of 1,002, 9 of 3,000 and 126 of 1,512 gainers changed gain band, and 338, 2,920 and 1,300 changed amount; no non-gainer moved. Every BADR band's offset changes, so each band's walk selects its claimants at new positions. On three synthetic frames of equal weights no claimant row of the old walk claims in the new one, while each frame's claimant count moved by one or two; the main asset types moved with the claims. ISA amounts do not move: on three synthetic frames every ISA output column is unchanged. The stage contracts of both stages, the H2 fixture's declarations and the 22 values of its synthetic asset-type facts (which the fixture tool sizes to the frame after the redraw), the coverage manifest's 25 source digests and `source_stages.json` are regenerated. Every node re-keys, since `graph_kernels` and `stochastic_assignment` changed and the spine's root binds `stochastic_assignment`; values move from stage 44 on, and `isa_subscriptions` re-keys through its `cgt_imputation` binding and its own module with no change in its output. diff --git a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py index dee464c78d..c72635e69c 100644 --- a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py +++ b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py @@ -17,6 +17,7 @@ "clipped_normal_from_uniforms", "integer_record_keys", "sample_categorical_from_counts", + "stable_identity_uniform", "stable_identity_uniforms", "weight_balanced_uniforms", ] @@ -48,6 +49,23 @@ def stable_identity_uniforms( ) +def stable_identity_uniform(key: object, *, seed: int, salt: str) -> float: + """One U[0,1) draw keyed by ``seed:salt:key``, for a label, not a record. + + The draw ``stable_identity_uniforms([key], seed=seed, salt=salt)`` gives, + for a scalar a stage keys on a label of its own (a walk's cell, a band's + bounds) rather than on a record id: it is the same whichever other draws + the stage makes, and in whatever order, where the n-th draw of a shared + Generator belongs to whichever call happens to come n-th (microcosm#931). + ``key`` is one value; a sequence would be formatted whole into the key + string, so it is refused. + """ + + if np.ndim(key) != 0: + raise TypeError("key must be a single value, not a sequence.") + return float(stable_identity_uniforms([key], seed=seed, salt=salt)[0]) + + def weight_balanced_uniforms( strata: Sequence[object] | np.ndarray, draws: Sequence[float] | np.ndarray, diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 66b2f5ad38..2ae9749a85 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "87879d883a622196c6176e728938d1b9738d1d58f1ab5ab2cbb11a2b5dd885eb", + "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index eae1a1ef56..c1fd6260ad 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -7643,7 +7643,7 @@ "mean_repair_reason": "Published counts round to the nearest thousand and amounts to the nearest million; two cells of the 2024-25 table (gains of GBP 1m to 2m at taxable incomes of GBP 37,700 to 49,999 and GBP 100,000 to 125,139) imply a mean below their own band, and repaired means clamp just inside the violated boundary.", "bottom_band_floor": "annual exempt amount plus one pound", "seed_base": 552, - "seed_mixing": "seed combined with the build period; each walk draws its rounding offset in cell order then pooled-fallback order, then each plan draws its quantile jitter, income bands ascending and gain bands highest first", + "seed_mixing": "seed combined with the build period, which salts every draw; each walk's rounding offset and each plan's quantile jitter is an identity-keyed uniform on its own label, with no stream order, so none depends on which other walks run or skip: a cell walk keys on cell::: and an income band's pooled walk on pool:, salted hmrc_cgt_gains_spine:offset:, and a plan on plan::, salted hmrc_cgt_gains_spine:jitter:, bands and age groups by their lower bounds and region groups by their labels", "deterministic": true, "cell_means": "ratios of the published Table 3 2024-25 cell gains to cell taxpayers, both rounded by the publisher (counts to the nearest thousand, amounts to the nearest million), so derived rather than published numbers; suppressed-count cells take the band-total mean; every mean is repaired into its band" }, @@ -7655,7 +7655,7 @@ "resource_vintage": "2017-18 nominal pounds as published (Advani and Summers 2020, CAGE WP 465, Table A1); no uprating is applied", "banding": "Advani-Summers total-income bands at the person's component-sum income without allowance subtraction, the banding the prior draw uses", "range": "quantiles strictly between the band's degree-1 quantile spline zero crossing and its crossing of the build period's annual exempt amount, so every amount lies in (0, AEA]", - "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; no seeded draw is consumed, so the walk and plan draw order is unchanged", + "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping draws nothing", "ordering": "existing gains ascending, person_id descending on ties (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key)", "quantile_points": [ 0.05, @@ -7926,7 +7926,7 @@ "qualifying_amount": "below the lifetime limit a claimant's whole net gain qualifies; in the open top band the qualifying gain is the lifetime limit", "top_band_count": "the published top-band qualifying gains divided by the lifetime limit; the published count is rounded to the nearest thousand", "basis_assumption": "Table 4.1 counts qualifying gains before losses and the annual exempt amount while the frame carries net gains after losses; Investors' Relief is merged with BADR, as in the engine's capital_gains_badr input; the whole-gain assumption reproduces Table 4.1's tax column to within half a percent", - "realization": "weighted systematic sampling within each band in ascending (gain, person_id) order with one seeded offset per band, drawn in band order whether or not the band carries a target", + "realization": "weighted systematic sampling within each band in ascending (gain, person_id) order with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare", "seed": 555, "seed_mixing": "seed combined with the build period", "policy_parameters": [ diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 6a071b24ba..48cc7f8eab 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -6213,7 +6213,7 @@ stages: mean_repair_reason: Published counts round to the nearest thousand and amounts to the nearest million; two cells of the 2024-25 table (gains of GBP 1m to 2m at taxable incomes of GBP 37,700 to 49,999 and GBP 100,000 to 125,139) imply a mean below their own band, and repaired means clamp just inside the violated boundary. bottom_band_floor: annual exempt amount plus one pound seed_base: 552 - seed_mixing: seed combined with the build period; each walk draws its rounding offset in cell order then pooled-fallback order, then each plan draws its quantile jitter, income bands ascending and gain bands highest first + seed_mixing: 'seed combined with the build period, which salts every draw; each walk''s rounding offset and each plan''s quantile jitter is an identity-keyed uniform on its own label, with no stream order, so none depends on which other walks run or skip: a cell walk keys on cell::: and an income band''s pooled walk on pool:, salted hmrc_cgt_gains_spine:offset:, and a plan on plan::, salted hmrc_cgt_gains_spine:jitter:, bands and age groups by their lower bounds and region groups by their labels' deterministic: true cell_means: ratios of the published Table 3 2024-25 cell gains to cell taxpayers, both rounded by the publisher (counts to the nearest thousand, amounts to the nearest million), so derived rather than published numbers; suppressed-count cells take the band-total mean; every mean is repaired into its band - kind: sub_aea_remainder @@ -6223,7 +6223,7 @@ stages: resource_vintage: 2017-18 nominal pounds as published (Advani and Summers 2020, CAGE WP 465, Table A1); no uprating is applied banding: Advani-Summers total-income bands at the person's component-sum income without allowance subtraction, the banding the prior draw uses range: quantiles strictly between the band's degree-1 quantile spline zero crossing and its crossing of the build period's annual exempt amount, so every amount lies in (0, AEA] - mapping: remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; no seeded draw is consumed, so the walk and plan draw order is unchanged + mapping: remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping draws nothing ordering: existing gains ascending, person_id descending on ties (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key) quantile_points: - 0.05 @@ -6440,7 +6440,7 @@ stages: qualifying_amount: below the lifetime limit a claimant's whole net gain qualifies; in the open top band the qualifying gain is the lifetime limit top_band_count: the published top-band qualifying gains divided by the lifetime limit; the published count is rounded to the nearest thousand basis_assumption: Table 4.1 counts qualifying gains before losses and the annual exempt amount while the frame carries net gains after losses; Investors' Relief is merged with BADR, as in the engine's capital_gains_badr input; the whole-gain assumption reproduces Table 4.1's tax column to within half a percent - realization: weighted systematic sampling within each band in ascending (gain, person_id) order with one seeded offset per band, drawn in band order whether or not the band carries a target + realization: weighted systematic sampling within each band in ascending (gain, person_id) order with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare seed: 555 seed_mixing: seed combined with the build period policy_parameters: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py index 1a2ffcecf4..93fb2dae7b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py @@ -56,9 +56,10 @@ like the residential flag: a logistic in centred log gains whose intercept matches the band's published claimant count and whose slope matches its published qualifying gains, realised by the same weighted - systematic walk with one seeded offset per band. Each band's logistic - carries the same stock shift for a gainer who shows business stock - (household corporate wealth, or self-employment income). The open top + systematic walk with one identity-keyed offset per band, keyed on its + bounds. Each band's logistic carries the same stock shift for a gainer + who shows business stock (household corporate wealth, or + self-employment income). The open top band holds claimants at the lifetime limit: its probability is flat in gains and varies only with the stock shift, its intercept matches the count, the count is the published gains over the limit (the published @@ -105,6 +106,7 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import stable_identity_uniform from microcosm.build.uk_runtime.cgt_imputation import ( UKCGTPolicyParameters, uk_cgt_policy_parameters, @@ -256,7 +258,7 @@ #: The BADR seed is combined with the build period, as the amounts stage #: does; distinct from the amounts stage's 552 so the two stages never share -#: a stream. The residential flag draws nothing since microcosm#1063: the +#: a draw. The residential flag draws nothing since microcosm#1063: the #: ``cgt_residential_split`` stage carries its probability as weight. CGT_BADR_FLAG_SEED = 555 @@ -1005,6 +1007,22 @@ def _json_number(value: float) -> float | None: return float(value) if np.isfinite(value) else None +def _badr_band_offset(band: HMRCCGTBADRBand, *, seed: int, time_period: int) -> float: + """A Table 4.1 band's walk offset: an identity-keyed uniform on its bounds. + + Keyed on the band's lower and upper edges, salted with the build period, + not on its position in the roster, so a facts revision that adds or + drops a band leaves every other band's offset alone (microcosm#931). + """ + + upper = "open" if band.upper_bound is None else band.upper_bound + return stable_identity_uniform( + f"{band.lower_bound}|{upper}", + seed=seed, + salt=f"assign_badr_qualifying_gains:offset:{int(time_period)}", + ) + + def _assign_badr_claims( *, gains: np.ndarray, @@ -1021,9 +1039,9 @@ def _assign_badr_claims( """Draw each Table 4.1 band's claimants and their qualifying gains. Returns the qualifying gain of every person (0 where not a claimant) and - one receipt row per band. One seeded offset per declared band is drawn - in band order whether or not the band is skipped, so the stream does - not depend on which bands carry a target. + one receipt row per band. Each band's walk offset is an identity-keyed + uniform on its bounds (``_badr_band_offset``), so it depends neither on + which bands carry a target nor on which other bands the facts declare. """ qualifying = np.zeros(gains.shape) @@ -1031,11 +1049,9 @@ def _assign_badr_claims( if not limit > 0.0: raise ValueError(f"BADR lifetime limit must be positive, got {limit}.") bands = facts.table4_bands - rng = np.random.default_rng((seed, int(time_period))) - offsets = rng.random(len(bands)) pool_gains = gains[pool_index] receipts: list[dict[str, object]] = [] - for position, band in enumerate(bands): + for band in bands: label = f"BADR band {band.label}" open_top = band.upper_bound is None if open_top: @@ -1111,7 +1127,10 @@ def _assign_badr_claims( amounts = band_gains order = np.lexsort((person_id[rows], band_gains)) flags = _weighted_systematic_flags( - probabilities, band_weights, order, float(offsets[position]) + probabilities, + band_weights, + order, + _badr_band_offset(band, seed=seed, time_period=time_period), ) claimants = rows[flags] qualifying[claimants] = amounts[flags] @@ -1931,8 +1950,12 @@ def cgt_asset_type_operation_parameters() -> dict[str, dict[str, Any]]: ), "realization": ( "weighted systematic sampling within each band in ascending " - "(gain, person_id) order with one seeded offset per band, drawn " - "in band order whether or not the band carries a target" + "(gain, person_id) order with one identity-keyed offset per " + "band, keyed on its bounds (|, open for the top band) and salted " + "assign_badr_qualifying_gains:offset:, with no stream " + "order, so a band's offset depends neither on which bands carry " + "a target nor on which other bands the facts declare" ), "seed": CGT_BADR_FLAG_SEED, "seed_mixing": "seed combined with the build period", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index 8c74c2752f..4c2b2b1f1a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -99,6 +99,7 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import stable_identity_uniform from microcosm.build.uk_runtime.advani_summers import ( ADVANI_SUMMERS_RESOURCE, ADVANI_SUMMERS_VINTAGE, @@ -168,8 +169,9 @@ #: the certified June H5) is retired; its family left the release contract. UK_CGT_SPINE_STAGE_NAME = "hmrc_cgt_gains_spine" -#: Base seed for the stage's draws. Combined with the build period so two -#: periods draw differently while each build is reproducible. +#: Base seed for the stage's draws. Combined with the build period, which +#: salts every walk offset and plan jitter, so two periods draw differently +#: while each build is reproducible. UK_CGT_IMPUTATION_SEED = 552 #: How the sub-AEA remainder takes its amounts (microcosm#970): the @@ -1224,12 +1226,30 @@ def margin_errors( return targets, report +class _KeyedOffset: + """One identity-keyed uniform in the shape the walks read (``random()``). + + ``_walk_bands`` reads its rounding offset, and ``_draw_plan_amounts`` its + quantile jitter, through one ``rng.random()`` call each. The redraw + passes each call one of these, holding a ``stable_identity_uniform`` + keyed on that walk's or plan's own label, in place of a shared + Generator, whose n-th draw belonged to whichever call came n-th + (microcosm#931). The ISA subscriptions walk their bands with it too. + """ + + def __init__(self, value: float) -> None: + self._value = float(value) + + def random(self) -> float: + return self._value + + def _walk_bands( *, ranked: np.ndarray, weights: np.ndarray, boundaries: Sequence[tuple[int, float]], - rng: np.random.Generator, + rng: np.random.Generator | _KeyedOffset, members: dict[int, list[np.ndarray]], ) -> tuple[np.ndarray, dict[int, float]]: """Assign ranked gainers to bands, highest first, and draw their amounts. @@ -1272,7 +1292,7 @@ def _draw_plan_amounts( person_id: np.ndarray, existing: np.ndarray, weights: np.ndarray, - rng: np.random.Generator, + rng: np.random.Generator | _KeyedOffset, new_gains: np.ndarray, ) -> None: """Draw one plan's amounts on weight-proportional quantile strata. @@ -1526,7 +1546,7 @@ def _placement_receipts( Receipts only: every input is what the two passes, the plan draws and the remainder mapping already produced, and nothing here touches - ``new_gains`` or the seeded stream. Placed rows are the gainers a pass + ``new_gains`` or draws. Placed rows are the gainers a pass assigned to a plan; demoted rows are the gainers left to the sub-exempt remainder. The initial band is the Table 2.1a band of the prior gain; the redrawn band is the band of the new amount, which for a placed row is the @@ -1787,7 +1807,16 @@ def impute_uk_capital_gains_with_report( # Receipts only (microcosm#1045): the walk never reads the row type. support_family, support_column_present = _support_family_rows(person, household) - rng = np.random.default_rng((seed, int(time_period))) + # Each walk's rounding offset and each plan's quantile jitter is an + # identity-keyed uniform on its own label, salted with the build period, + # so none depends on which other walks run or skip, or in what order + # (microcosm#931). + offset_salt = f"{UK_CGT_SPINE_STAGE_NAME}:offset:{int(time_period)}" + jitter_salt = f"{UK_CGT_SPINE_STAGE_NAME}:jitter:{int(time_period)}" + + def keyed(salt: str, key: str) -> _KeyedOffset: + return _KeyedOffset(stable_identity_uniform(key, seed=seed, salt=salt)) + new_gains = existing.copy() is_gainer = existing > 0 assigned = np.zeros(len(person), dtype=bool) @@ -1863,7 +1892,11 @@ def impute_uk_capital_gains_with_report( ranked=ranked, weights=weights, boundaries=boundaries, - rng=rng, + rng=keyed( + offset_salt, + f"cell:{income_lower}:{UK_CGT_AGE_GROUP_LOWER_BOUNDS[a]}:" + f"{UK_CGT_REGION_GROUP_LABELS[r]}", + ), members=cell_members, ) for gain_lower, chunks in cell_members.items(): @@ -1914,7 +1947,7 @@ def impute_uk_capital_gains_with_report( ranked=ranked, weights=weights, boundaries=boundaries, - rng=rng, + rng=keyed(offset_salt, f"pool:{income_lower}"), members=pool_members, ) for gain_lower, chunks in pool_members.items(): @@ -1937,14 +1970,13 @@ def impute_uk_capital_gains_with_report( person_id=person_id, existing=rank_key, weights=person_weight, - rng=rng, + rng=keyed(jitter_salt, f"plan:{gain_lower}:{income_lower}"), new_gains=new_gains, ) # Below the published taxpayer mass: sub-AEA gainers take Advani-Summers # within-band amounts restricted to (0, AEA], rank-preserving within each - # A&S total-income band (microcosm#970). Nothing here touches rng, so the - # walk and plan draw order above is unchanged. + # A&S total-income band (microcosm#970). Nothing here draws. remainder_receipt = _map_remainder_amounts( remainder=np.flatnonzero(is_gainer & ~assigned), existing=existing, @@ -2512,10 +2544,16 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: "bottom_band_floor": "annual exempt amount plus one pound", "seed_base": UK_CGT_IMPUTATION_SEED, "seed_mixing": ( - "seed combined with the build period; each walk draws its " - "rounding offset in cell order then pooled-fallback order, then " - "each plan draws its quantile jitter, income bands ascending and " - "gain bands highest first" + "seed combined with the build period, which salts every draw; " + "each walk's rounding offset and each plan's quantile jitter is " + "an identity-keyed uniform on its own label, with no stream " + "order, so none depends on which other walks run or skip: a cell " + "walk keys on cell::: and " + "an income band's pooled walk on pool:, salted " + "hmrc_cgt_gains_spine:offset:, and a plan on " + "plan::, salted " + "hmrc_cgt_gains_spine:jitter:, bands and age groups by " + "their lower bounds and region groups by their labels" ), "deterministic": True, "cell_means": ( @@ -2561,9 +2599,8 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: "remainder gainers take the midpoints of the weight-proportional " "quantile strata that partition the range, evaluated on the " "band's spline; zero-weight persons take a nominal stratum of " - "the band's smallest positive weight and carry no mass; no " - "seeded draw is consumed, so the walk and plan draw order is " - "unchanged" + "the band's smallest positive weight and carry no mass; the " + "mapping draws nothing" ), "ordering": ( "existing gains ascending, person_id descending on ties (the " diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 9a46a6bbb2..ba26a9efb6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -425,8 +425,9 @@ # rule from cgt_structure, and the residential split its logistic from # cgt_asset_type; the incidence clone sums cgt_imputation's income proxy # components into the income behind its propensity; the gains spine names - # HMRC's regional conditioning rows by microcosm.calibrate's UK region tier - # (microcosm#931). + # HMRC's regional conditioning rows by microcosm.calibrate's UK region tier, + # and it and the asset types key each walk's offset on its label through + # the shared keyed draws (microcosm#931). "cgt_support_split": ( copy_id_table, *_FEED_FACT_MODULES, @@ -470,11 +471,13 @@ hmrc_capital_gains, hmrc_income, target_materialization, + stochastic_assignment, ), "hmrc_cgt_asset_type_spine": ( *_FEED_FACT_MODULES, cgt_imputation, hmrc_capital_gains, + stochastic_assignment, ), # The legacy proxies read raw tabs, and the grant split resolves its # year's rules from the FRS release (microcosm#931). diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py index 37dafd7564..018a0f3778 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py @@ -61,8 +61,12 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.stochastic_assignment import ( + stable_identity_uniform, + stable_identity_uniforms, +) from microcosm.build.uk_runtime.cgt_imputation import ( + _KeyedOffset, _truncated_exponential_quantile, _walk_bands, uk_cgt_component_sum_income, @@ -199,18 +203,10 @@ class ISASubscriptionsError(ValueError): """Raised when the inputs or the declaration do not carry what the stage needs.""" -class _KeyedOffset: - """One identity-keyed offset in the shape ``_walk_bands`` reads (``random()``).""" - - def __init__(self, value: float) -> None: - self._value = float(value) - - def random(self) -> float: - return self._value - - def _keyed_offset(seed: int, salt: str, key: str) -> _KeyedOffset: - return _KeyedOffset(stable_identity_uniforms([key], seed=seed, salt=salt)[0]) + """One walk's offset, keyed on its label, in the shape ``_walk_bands`` reads.""" + + return _KeyedOffset(stable_identity_uniform(key, seed=seed, salt=salt)) # -------------------------------------------------------------------------- @@ -1064,9 +1060,9 @@ def draw_band_amounts( cumulative = np.cumsum(row_weights[ordered]) tops = cumulative / cumulative[-1] bottoms = np.concatenate(([0.0], tops[:-1])) - jitter = stable_identity_uniforms( - [f"{key}:{band}"], seed=seed, salt=f"{salt}:jitter" - )[0] + jitter = stable_identity_uniform( + f"{key}:{band}", seed=seed, salt=f"{salt}:jitter" + ) quantiles = bottoms + (0.05 + 0.9 * jitter) * (tops - bottoms) amounts[ordered] = lower + quantiles * (upper - lower) return amounts diff --git a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py index 1c34c4aab1..1603c9e9e4 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py @@ -11,6 +11,7 @@ clipped_normal_from_uniforms, integer_record_keys, sample_categorical_from_counts, + stable_identity_uniform, stable_identity_uniforms, weight_balanced_uniforms, ) @@ -132,6 +133,30 @@ def test_stable_identity_uniforms_is_permutation_equivariant() -> None: np.testing.assert_array_equal(permuted_draws[[1, 2, 0]], draws) +def test_stable_identity_uniform_is_the_array_helpers_draw_for_one_label() -> None: + # Labels of the kind the UK stages key a walk's offset on (microcosm#931). + for key, seed, salt in ( + ("cell:0:45:london", 552, "hmrc_cgt_gains_spine:offset:2024"), + ("0|10000", 555, "assign_badr_qualifying_gains:offset:2024"), + (1001, 7, "flag"), + ): + value = stable_identity_uniform(key, seed=seed, salt=salt) + assert type(value) is float + assert 0.0 <= value < 1.0 + assert value == stable_identity_uniforms([key], seed=seed, salt=salt)[0] + base = stable_identity_uniform("a", seed=0, salt="x") + assert base != stable_identity_uniform("a", seed=1, salt="x") + assert base != stable_identity_uniform("a", seed=0, salt="y") + assert base != stable_identity_uniform("b", seed=0, salt="x") + + +def test_stable_identity_uniform_refuses_a_sequence_for_its_label() -> None: + # A list would be formatted whole into the key and draw something else. + for key in (["a"], ("a", "b"), np.array([1, 2])): + with pytest.raises(TypeError, match="single value"): + stable_identity_uniform(key, seed=0, salt="x") + + def test_anchored_residual_population_restricts_the_draw_and_the_target() -> None: """The rate is a share of the population; rows outside it are never drawn.""" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py index d04fe942e5..794b43d455 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py @@ -10,6 +10,7 @@ import pytest from microcosm.build.country_spec import load_country_spec +from microcosm.build.stochastic_assignment import stable_identity_uniform from microcosm.build.uk_runtime import cgt_asset_type from microcosm.build.uk_runtime.cgt_asset_type import ( CGT_ASSET_TYPE_COLUMN, @@ -704,6 +705,77 @@ def test_is_deterministic_and_seed_sensitive(self) -> None: other[CGT_RESIDENTIAL_GAINS_COLUMN] ) + def test_an_inserted_band_moves_no_other_bands_offset_or_claims(self) -> None: + """Each band's walk offset is an identity-keyed uniform on its bounds + (microcosm#931), where it was the band's position in one draw of every + band's offset, so a facts revision that adds a band moved the offset + of every band after it.""" + + gains = _synthetic_gains() + weights = np.full(gains.size, 60.0) + pool = np.flatnonzero(gains > PARAMETERS.annual_exempt_amount) + facts = _synthetic_facts(gains, weights) + # The GBP 50,000 to 99,999 band split in two, each half claiming a + # tenth of its pool's mass at the pool's mean gain, as the rest do. + halves = [] + for lower, upper in ((50_000, 75_000), (75_000, 100_000)): + in_band = (gains[pool] >= lower) & (gains[pool] < upper) + mass = float(weights[pool][in_band].sum()) + mean = float((weights * gains)[pool][in_band].sum()) / mass + halves.append( + HMRCCGTBADRBand(lower, upper, 0.1 * mass, 0.1 * mass * mean, None) + ) + split = dataclasses.replace( + facts, + table4_bands=tuple( + band + for original in facts.table4_bands + for band in (halves if original.lower_bound == 50_000 else [original]) + ), + ) + + def claims(facts) -> tuple[np.ndarray, dict[tuple, float]]: + offsets: list[float] = [] + walk = cgt_asset_type._weighted_systematic_flags + + def recording(probabilities, band_weights, order, offset): + offsets.append(offset) + return walk(probabilities, band_weights, order, offset) + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(cgt_asset_type, "_weighted_systematic_flags", recording) + qualifying, receipts = cgt_asset_type._assign_badr_claims( + gains=gains, + person_weight=weights, + person_id=np.arange(gains.size), + pool_index=pool, + facts=facts, + badr_parameters=BADR_PARAMETERS, + annual_exempt_amount=PARAMETERS.annual_exempt_amount, + time_period=2024, + seed=555, + ) + walked = [receipt for receipt in receipts if not receipt["skipped"]] + assert len(walked) == len(offsets) == len(facts.table4_bands) + bounds = [(row["lower_bound"], row["upper_bound"]) for row in walked] + return qualifying, dict(zip(bounds, offsets, strict=True)) + + before, offsets = claims(facts) + after, split_offsets = claims(split) + + kept = set(offsets) - {(50_000, 100_000)} + assert set(split_offsets) == kept | {(50_000, 75_000), (75_000, 100_000)} + for bounds in kept: + assert split_offsets[bounds] == offsets[bounds], bounds + for (lower, upper), offset in split_offsets.items(): + key = f"{lower}|{'open' if upper is None else upper}" + salt = "assign_badr_qualifying_gains:offset:2024" + assert offset == stable_identity_uniform(key, seed=555, salt=salt) + # The other bands' claimants and qualifying gains are as they were. + outside = (gains < 50_000) | (gains >= 100_000) + np.testing.assert_array_equal(after[outside], before[outside]) + assert (before[outside] > 0.0).any() + def test_refuses_claims_the_eligible_types_cannot_hold(self) -> None: tiny = tuple( ( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py index 06d71dde44..ba370d8c02 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py @@ -1,6 +1,8 @@ """Tests split from packages/microcosm-build/tests/test_uk_cgt_imputation.py.""" # ruff: noqa: F403, F405 +from microcosm.build.stochastic_assignment import stable_identity_uniform +from microcosm.build.uk_runtime.cgt_imputation import UK_CGT_IMPUTATION_SEED from test_support.microcosm_build.uk_cgt_imputation import * @@ -1482,3 +1484,263 @@ def test_refuses_a_copy_count_below_one(self) -> None: ) with pytest.raises(ValueError, match="copy count"): impute_uk_capital_gains(broken, _distribution(), PARAMETERS) + + +#: (employment income, age, region, gainers) per cell, gainers weighing 100 +#: each against a synthetic joint of 300 people per (gain band, income band) +#: cell: three cells in the lowest income band, whose 65-74 cell is short of +#: its raked target so the band's pooled walk runs, and one cell in each of +#: two higher bands. One cell per band leaves the rake's final scaling, which +#: holds the joint, nothing to split there, so those bands' targets do not +#: move with the lowest band's cells. +_KEYED_CELLS = ( + (20_000.0, 30, "NORTH_EAST", 20), + (20_000.0, 45, "LONDON", 60), + (20_000.0, 70, "LONDON", 5), + (55_000.0, 50, "SOUTH_EAST", 40), + (70_000.0, 60, "WALES", 40), +) +#: The cells' labels: income band and age group by their lower bounds, then +#: the region group by its label. +_KEYED_CELL_LABELS = ( + "cell:0:25:rest_of_england", + "cell:0:45:london", + "cell:0:65:london", + "cell:37700:45:south_east_and_east_of_england", + "cell:50000:55:wales_scotland_northern_ireland", +) +#: A cell that walks before every other one in the lowest income band. +_KEYED_EARLY_CELL = (20_000.0, 20, "WALES", 15) + + +def _cells_frame(cells, *, zeroed=()) -> Frame: + """One gainer per household for each ``(income, age, region, count)`` cell. + + Each cell draws its gains from its own seeded stream and keeps its rows' + person ids, so appending a cell or zeroing one (``zeroed`` names cells by + position; their rows hold no gains) leaves every other row as it was. + """ + + gains, incomes, ages, regions = [], [], [], [] + for index, (income, age, region, count) in enumerate(cells): + drawn = np.random.default_rng((7, index)).lognormal(10.0, 1.2, count) + gains.extend(np.zeros(count) if index in zeroed else drawn) + incomes.extend([income] * count) + ages.extend([age] * count) + regions.extend([region] * count) + rows = len(gains) + return _frame( + rows, + gains=gains, + incomes=incomes, + ages=ages, + regions=regions, + weights=np.full(rows, 100.0), + ) + + +class _Peek: + """Hands a call its rng and records each uniform the call reads from it.""" + + def __init__(self, inner) -> None: + self.inner = inner + self.values: list[float] = [] + + def random(self) -> float: + value = float(self.inner.random()) + self.values.append(value) + return value + + +def _labelled_draws(frame: Frame): + """Run the redraw, recording each walk's and plan's uniforms by label. + + A label comes from the call's own arguments, never from the uniform it + is handed: a walk over the whole gainer set of a cell no walk has covered + is that cell's walk, every later walk is its income band's pooled walk, + and a plan is its gain band and its members' income band. Returns the new + gains by person id and ``{label: (uniforms read, member person ids)}``. + """ + + person = frame.table("person") + regions = frame.table("household").set_index("household_id")["region"] + region = person["person_household_id"].map(regions).to_numpy() + proxy = uk_cgt_taxable_income_proxy(person, PARAMETERS) + income = np.asarray(HMRC_CGT_INCOME_BAND_LOWER_BOUNDS)[ + np.digitize(proxy, HMRC_CGT_INCOME_BAND_LOWER_BOUNDS[1:]) + ] + age = np.asarray(UK_CGT_AGE_GROUP_LOWER_BOUNDS)[ + np.digitize(person["age"].to_numpy(), UK_CGT_AGE_GROUP_LOWER_BOUNDS[1:]) + ] + cell = np.asarray( + [ + f"{band}:{group}:{UK_CGT_REGION_GROUPS[name]}" + for band, group, name in zip(income, age, region, strict=True) + ] + ) + gainer = person["capital_gains"].to_numpy() > 0.0 + person_id = person["person_id"].to_numpy() + walk = cgt_imputation._walk_bands + plan = cgt_imputation._draw_plan_amounts + draws: dict[str, tuple[list[float], np.ndarray]] = {} + pooled = False + + def band_of(members: np.ndarray) -> str: + (band,) = {label.split(":")[0] for label in cell[members]} + return band + + def record(label: str, peek: _Peek, members: np.ndarray) -> None: + assert label not in draws, label + draws[label] = (peek.values, np.sort(person_id[members])) + + def recording_walk(**kwargs): + nonlocal pooled + peek = _Peek(kwargs["rng"]) + result = walk(**{**kwargs, "rng": peek}) + members = np.asarray(kwargs["ranked"]) + first = cell[members[0]] + whole = np.flatnonzero(gainer & (cell == first)) + if ( + not pooled + and f"cell:{first}" not in draws + and np.array_equal(np.sort(members), whole) + ): + record(f"cell:{first}", peek, members) + else: + pooled = True + record(f"pool:{band_of(members)}", peek, members) + return result + + def recording_plan(plan_cell, members, **kwargs): + peek = _Peek(kwargs["rng"]) + plan(plan_cell, members, **{**kwargs, "rng": peek}) + members = np.asarray(members) + record(f"plan:{plan_cell.gain_lower_bound}:{band_of(members)}", peek, members) + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(cgt_imputation, "_walk_bands", recording_walk) + patch.setattr(cgt_imputation, "_draw_plan_amounts", recording_plan) + result, _ = impute_uk_capital_gains_with_report( + frame, + _distribution(cell_people=300.0), + PARAMETERS, + conditioning=load_hmrc_cgt_conditioning_facts(), + ) + drawn = result.table("person") + gains = pd.Series( + drawn["capital_gains"].to_numpy(), index=drawn["person_id"].to_numpy() + ).sort_index() + return gains, draws + + +def _label_band(label: str) -> str: + """The income band a walk's or plan's label names.""" + + kind, *parts = label.split(":") + return parts[1] if kind == "plan" else parts[0] + + +class TestKeyedWalkOffsets: + """Each walk's offset and each plan's jitter is an identity-keyed uniform + on its own label, not the next draw of one shared stream (microcosm#931). + + The keying fixes the uniforms and nothing more. The rounding carry runs + through an income band's cells, the pooled walk takes the band's net + shortfall over its unassigned gainers, and a plan stratifies members + pooled from every cell, so a change in one cell can still move the + assignments and amounts of other cells in its income band, and a plan's + amounts whenever its members change. These tests assert the uniforms the + keying fixes, and members and amounts only in income bands whose targets + the change leaves alone. + """ + + def test_each_walk_and_plan_reads_one_uniform_keyed_on_its_label(self) -> None: + _, draws = _labelled_draws(_cells_frame(_KEYED_CELLS)) + + cells = {label for label in draws if label.startswith("cell:")} + assert cells == set(_KEYED_CELL_LABELS) + assert {label for label in draws if label.startswith("pool:")} == {"pool:0"} + plans = {label for label in draws if label.startswith("plan:")} + assert {_label_band(label) for label in plans} == {"0", "37700", "50000"} + for label, (uniforms, _) in draws.items(): + kind = "jitter" if label.startswith("plan:") else "offset" + salt = f"hmrc_cgt_gains_spine:{kind}:2023" + assert uniforms == [ + stable_identity_uniform(label, seed=UK_CGT_IMPUTATION_SEED, salt=salt) + ], label + + def test_the_build_period_salts_every_offset_and_jitter(self, monkeypatch) -> None: + frame = _cells_frame(_KEYED_CELLS) + _, before = _labelled_draws(frame) + monkeypatch.setattr(cgt_imputation, "uk_time_period", lambda frame: 2024) + _, after = _labelled_draws(frame) + + shared = set(before) & set(after) + assert set(_KEYED_CELL_LABELS) <= shared + for label in shared: + kind = "jitter" if label.startswith("plan:") else "offset" + salt = f"hmrc_cgt_gains_spine:{kind}:2024" + assert after[label][0] == [ + stable_identity_uniform(label, seed=UK_CGT_IMPUTATION_SEED, salt=salt) + ], label + assert after[label][0] != before[label][0], label + + @pytest.mark.parametrize( + "change", ["an earlier cell gains members", "a cell skips"] + ) + def test_a_uniform_holds_when_other_cells_skip_or_gain_members( + self, change: str + ) -> None: + """Either change adds or drops a walk before the others in the lowest + income band, which moved every later draw of the shared stream; a + keyed uniform stays with its label. The higher income bands' cells + and plans then take the same members and amounts, since the rake + leaves those one-cell bands' targets where they were.""" + + base_gains, base = _labelled_draws(_cells_frame(_KEYED_CELLS)) + if change == "an earlier cell gains members": + frame = _cells_frame((*_KEYED_CELLS, _KEYED_EARLY_CELL)) + added, dropped = {"cell:0:16:wales_scotland_northern_ireland"}, set() + else: + frame = _cells_frame(_KEYED_CELLS, zeroed=(0,)) + added, dropped = set(), {"cell:0:25:rest_of_england"} + gains, draws = _labelled_draws(frame) + + cells = {label for label in draws if label.startswith("cell:")} + assert cells == (set(_KEYED_CELL_LABELS) | added) - dropped + shared = set(draws) & set(base) + # The walks after the change, the pooled walk among them, still read. + assert set(_KEYED_CELL_LABELS[1:]) - dropped <= shared + assert "pool:0" in shared + for label in shared: + assert draws[label][0] == base[label][0], label + # Nothing in the higher income bands moves. + higher = {label for label in base if _label_band(label) != "0"} + assert {label for label in draws if _label_band(label) != "0"} == higher + for label in higher: + members = base[label][1] + np.testing.assert_array_equal(draws[label][1], members, err_msg=label) + np.testing.assert_array_equal( + gains[members].to_numpy(), base_gains[members].to_numpy(), err_msg=label + ) + + def test_the_redraw_does_not_depend_on_the_row_order(self) -> None: + frame = _cells_frame((*_KEYED_CELLS, _KEYED_EARLY_CELL)) + person = frame.table("person") + order = np.random.default_rng(5).permutation(len(person)) + shuffled = uk_national_frame( + person=person.iloc[order].reset_index(drop=True), + benunit=frame.table("benunit"), + household=frame.table("household"), + time_period="2023", + household_weights=frame.weights_for("household").values, + ) + + gains, draws = _labelled_draws(frame) + shuffled_gains, shuffled_draws = _labelled_draws(shuffled) + + pd.testing.assert_series_equal(shuffled_gains, gains, check_exact=True) + assert set(shuffled_draws) == set(draws) + for label, (uniforms, members) in draws.items(): + assert shuffled_draws[label][0] == uniforms, label + np.testing.assert_array_equal(shuffled_draws[label][1], members) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 8cb38dbf6f..d23cea14dd 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -670,6 +670,9 @@ def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( ) # Stage 14 draws through the employer pension helper, which it binds too. assert "frs_hmrc_spine_leaves" in stages + # The CGT redraw and the asset types key their walk offsets on labels + # through the helpers (microcosm#931). + assert {"hmrc_cgt_gains_spine", "hmrc_cgt_asset_type_spine"} <= set(stages) # The root stage's node is the CREATE kernel. affected = [ graph.node("create_uk_frs" if stage == "frs_spine" else stage).kernel @@ -678,8 +681,8 @@ def test_stochastic_assignment_changes_invalidate_every_stage_that_draws( # Three stages that draw through nothing of it. control_stages = ( "regional_home_values", - "hmrc_cgt_gains_spine", - "hmrc_cgt_asset_type_spine", + "regional_property_uprating", + "private_school_attendance", ) drawing = _stages_drawing_through_stochastic_assignment() assert not set(control_stages) & drawing diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py index 0ed79b1791..6248b19609 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py @@ -425,6 +425,38 @@ def test_reordering_the_rows_moves_no_draw(statistics): ) +def test_the_walk_offsets_and_jitters_are_the_uniforms_they_were(spec): + """The keyed offset moved to the shared scalar and the redraw's adapter + (microcosm#931); every walk offset and band jitter is the uniform the + stage keyed before the move, bit for bit.""" + + from microcosm.build import stochastic_assignment + from microcosm.build.uk_runtime import cgt_imputation + + assert stage._KeyedOffset is cgt_imputation._KeyedOffset + assert ( + stage.stable_identity_uniform is stochastic_assignment.stable_identity_uniform + ) + before = stochastic_assignment.stable_identity_uniforms + for isa_type in stage.CLASS_ORDER: + # The selection's cell walks, then the amounts' walk and band jitter, + # salted as the stage salts them. + select = f"{spec.select_salt}:{isa_type}" + amount = f"{spec.amount_salt}:{isa_type}" + for seed, salt, key in ( + (spec.select_seed, f"{select}:cell", "income_3|age_2|female|london"), + (spec.amount_seed, f"{amount}:walk", "income_3"), + ): + offset = stage._keyed_offset(seed, salt, key) + assert isinstance(offset, cgt_imputation._KeyedOffset) + assert offset.random() == before([key], seed=seed, salt=salt)[0] + salt = f"{amount}:jitter" + jitter = stage.stable_identity_uniform( + "income_3:2", seed=spec.amount_seed, salt=salt + ) + assert jitter == before(["income_3:2"], seed=spec.amount_seed, salt=salt)[0] + + def test_the_both_split_reports_saturation(): combined = np.full(10, 10_000.0) weights = np.ones(10) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json index be9eb32626..a69be24ef5 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json @@ -2,68 +2,68 @@ "resource": "synthetic-cgt-asset-type.json", "resource_sha256": "synthetic", "source_commit": "synthetic", - "table4_all_gains": 7278689.42603867, + "table4_all_gains": 6698701.203441119, "table4_all_tax": 0.0, "table4_all_taxpayers": 17.24436988246394, "table4_bands": [ { - "gains": 7949.502620452311, + "gains": 6998.611130126762, "lower_bound": 0, "tax": null, - "taxpayers": 1.9673816546685436, + "taxpayers": 1.8896108992721439, "upper_bound": 10000 }, { - "gains": 26946.918634102105, + "gains": 37848.09165770648, "lower_bound": 10000, "tax": null, - "taxpayers": 1.7272922430098223, + "taxpayers": 2.3468115638209706, "upper_bound": 25000 }, { - "gains": 51102.16729150777, + "gains": 64412.5306127065, "lower_bound": 25000, "tax": null, - "taxpayers": 1.3784835361616843, + "taxpayers": 1.887044121541452, "upper_bound": 50000 }, { - "gains": 144887.97165642734, + "gains": 106965.85772030169, "lower_bound": 50000, "tax": null, - "taxpayers": 2.0034797890802527, + "taxpayers": 1.5589961305052387, "upper_bound": 100000 }, { - "gains": 289261.98743372096, + "gains": 247212.82520699556, "lower_bound": 100000, "tax": null, - "taxpayers": 1.7329974891738054, + "taxpayers": 1.5695959119641265, "upper_bound": 250000 }, { - "gains": 554667.0416390727, + "gains": 593976.5902481461, "lower_bound": 250000, "tax": null, - "taxpayers": 1.6027496904998344, + "taxpayers": 1.810480713045304, "upper_bound": 500000 }, { - "gains": 1628625.8536635868, + "gains": 1466550.0712794645, "lower_bound": 500000, "tax": null, - "taxpayers": 2.2567374967701963, + "taxpayers": 2.007093916729033, "upper_bound": 1000000 }, { - "gains": 4575247.983099801, + "gains": 4174736.6255856715, "lower_bound": 1000000, "tax": null, - "taxpayers": 4.575247983099801, + "taxpayers": 4.1747366255856715, "upper_bound": null } ], - "table4_individuals_gains": 7278689.42603867, + "table4_individuals_gains": 6698701.203441119, "table4_individuals_tax": 0.0, "table4_individuals_taxpayers": 17.24436988246394, "table4_trusts_gains": 0.0, @@ -115,11 +115,11 @@ } ], "table8a_disposals_total": 32.51795463550343, - "table8a_gains_total": 43141212.12856026, - "table8a_tax_total": 8628242.425712053, + "table8a_gains_total": 37770677.36531601, + "table8a_tax_total": 7554135.473063203, "table8a_taxpayers_total": 29.56177694136675, - "table8b_all_gains": 43141212.12856026, + "table8b_all_gains": 37770677.36531601, "table8b_all_taxpayers": 29.56177694136675, - "table8b_individuals_gains": 43141212.12856026, + "table8b_individuals_gains": 37770677.36531601, "table8b_individuals_taxpayers": 29.56177694136675 } diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index faec8df43e..85de0d0722 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3175,7 +3175,7 @@ ], "population": "liable gainers whose arm is not residential (capital_gains_residential_property == 0, cgt_residential_split)", "qualifying_amount": "below the lifetime limit a claimant's whole net gain qualifies; in the open top band the qualifying gain is the lifetime limit", - "realization": "weighted systematic sampling within each band in ascending (gain, person_id) order with one seeded offset per band, drawn in band order whether or not the band carries a target", + "realization": "weighted systematic sampling within each band in ascending (gain, person_id) order with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare", "reported_not_fitted": "tax at the relief rate, with the annual exempt amount set against the main-rate gains first as the engine allocates it, against Table 4.1's tax column", "seed": 555, "seed_mixing": "seed combined with the build period", @@ -3408,14 +3408,14 @@ "open_band_realization": "each open-band person takes the conditional mean of their weight-proportional quantile stratum, which averages back to the published mean exactly under the household weights; an iid draw from a Pareto with infinite variance sits well below it on a few dozen carriers, and equal strata on unequal weights put the tail's mass on whichever rows happen to be light", "quantile_scheme": "stratified by weight: the persons a (gain band, income band) plan receives across every cell and the pooled walk take the quantile strata that partition (0, 1) in proportion to their household weights, in ascending rank-key order with one seeded jitter per plan, so the plan's weighted realised mean sits on its published mean rather than carrying n independent draws or the weight-rank mix of equal strata", "seed_base": 552, - "seed_mixing": "seed combined with the build period; each walk draws its rounding offset in cell order then pooled-fallback order, then each plan draws its quantile jitter, income bands ascending and gain bands highest first" + "seed_mixing": "seed combined with the build period, which salts every draw; each walk's rounding offset and each plan's quantile jitter is an identity-keyed uniform on its own label, with no stream order, so none depends on which other walks run or skip: a cell walk keys on cell::: and an income band's pooled walk on pool:, salted hmrc_cgt_gains_spine:offset:, and a plan on plan::, salted hmrc_cgt_gains_spine:jitter:, bands and age groups by their lower bounds and region groups by their labels" }, { "banding": "Advani-Summers total-income bands at the person's component-sum income without allowance subtraction, the banding the prior draw uses", "deterministic": true, "extrapolation": "ext=0", "kind": "sub_aea_remainder", - "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; no seeded draw is consumed, so the walk and plan draw order is unchanged", + "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping draws nothing", "ordering": "existing gains ascending, person_id descending on ties (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key)", "policy": "gainers beyond the published taxpayer mass take amounts from the Advani-Summers within-band gains distribution restricted to (0, annual exempt amount], rank-preserving within each Advani-Summers total-income band", "quantile_points": [ diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 9d7e2f13f4..a9ec1d0364 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"ed46fd4d2dc5df11ea9fdd0e955d8a7e13c20be5bae275b7a0a2f209a653fc78","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"0edd2cdc89dbbb4fc9ad5332db4484522904f3bcede684cb89b189e99eafc537","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"d70c2bf8939caef93e4449bf86065c7e15586696eb0c832b0589f4f87efedd5f","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"e6423b26245ec3915071a92c39a45c4585371ffdab91e4576263b08819da31f5","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"ed46fd4d2dc5df11ea9fdd0e955d8a7e13c20be5bae275b7a0a2f209a653fc78","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"0edd2cdc89dbbb4fc9ad5332db4484522904f3bcede684cb89b189e99eafc537","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"660d72066059c354477e197f3c10cd27b6d3b91d433e3dc06c5551724a0a46ba","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"5e1c540675d3582a2840bc38f37600a0f6ce552471e6a2165c7705a6edc60a80","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} From cca9a16a6cd88a57f653fceaf47e1dcaf16d4b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:19:23 +0000 Subject: [PATCH 041/108] Break ties at the six id-ordered walks on identity-keyed uniforms (#931) Six walks ordered exactly tied rows by id: stage 42's support walk, stage 26's high-value stratum, stage 28's receipts walk, stage 44's rank and stage 47's BADR and main asset type walks. Ties broken by id alone put the lowest id first every time, and the copies and geographic clones of one household differ only in their copy ordinal, so the first copy or clone won every tie (synthetic probe of the design: the lower clone index ranked first in all of 70,035 tied clone pairs under id ties, in 49.9% under keyed ties). Accepted as designed on 7 Nov, with the verifier corrections. - Helper: stochastic_assignment.keyed_tie_order(primary, ids, *, seed, salt) returns np.lexsort((ids, u, primary)), u the records' keyed_tie_uniforms(ids, *, seed, salt), stable_identity_uniforms of integer_record_keys(ids, label=salt). It is a secondary sort key, the form stage 27 uses, so distinct values keep the id order bit for bit; it draws only the tied records' uniforms. It keys each row on its own id, never on spine_household_keys, and is not the rank-inside-a-tie form. - Sites: stage 42 seed 0, salt cgt_support_split:wealth_tie; stage 26 seed 0, salt high_value_home_support:value_tie, its mid-rank unchanged; stage 28 walk_seed and a new tie_salt allocate_property_rental_income:tie, which ReceiptsAllocationSpec reads and refuses when empty or equal to a salt the stage's finance-cost draws key the same person ids on (the op has no other salt, so the stage passes those two); stage 44 one uniform per person, seed 552, salt hmrc_cgt_gains_spine:rank_tie:, drawn once beside rank_key and passed to _ranked, _draw_plan_amounts and _map_remainder_amounts as a keyword-only tie (None keeps the person_id order, so their direct test calls are unchanged); stage 47 seed 555, salts assign_badr_qualifying_gains:tie: and assign_main_asset_type:tie:. - Declarations: the two split operations declare seed and salt and the receipts operation its tie_salt, in the code dictionaries, the strict schema (three operations), sources.yaml and source_stages.json. The selection, stratum_rule, order, ordering, mapping, realization and values texts, the two stage notes, the docstrings and the receipts walk's and asset types' receipt order texts follow; "no draw, no seed, no salt" and "deterministic, no seed" are gone. spine_build._declared_seeds registers the splits under their salts and walk_seed under the tie salt. - Rule B at stage 42: both selections take seed and salt with no default. The split, keeps_the_split's rerun and the identity tool's recompute pass cgt_support_tie_break(), read from the declared operation. The exact-total correction's candidate order (the heaviest whole households first, tied weights in row order) stays: it places a rounding-sized correction, not a draw, and copy_rule states it. - One stage 42 walk on a synthetic vector the size of a K = 15 spine takes 0.5 to 0.7 s, against 0.38 s before (the uniforms of tied households and the id check); the correction cap's comment gives the new cost. - The five stages already bind stochastic_assignment, so the derived binding rule needs nothing new. No receipt key is added; the band receipt's wealth_threshold is untouched. - Stale, left for (c): uk/gates.json's high-value home support gate note says the stage "makes no draw". gates.json is not edited here. What moves (synthetic counts only): nothing where no exact tie touches a cut or a position schedule. - Stage 42: a band's crossing household in a tie group. With three clones per household, 2 to 8 of about 850 households per frame changed copy count over six frames, and clone 0, which carried the most families in all six, now does in one; at one clone none moved (the tied family members there were too light to copy). - Stage 26: the tie group across a region's cut, 0 to 5 of 3,000 owners per frame over six frames of values on 300 atoms. - Stage 28: receipts among tied landlords; about two thirds of the tied landlords' receipts moved on three frames, none outside the tie groups, band counts unchanged. - Stage 44: 43, 99 and 62 of about 1,800 gainers moved amount where wealth varies (the blended key ties), none where wealth is equal; no non-gainer moved. - Stage 47: bit for bit on three frames. - Node keys: every spine node re-keys, since stochastic_assignment changed and the root binds it; 49 of the 79 nodes' own implementation hashes move. Values move from stage 26 on. - Regenerated: source_stages.json; the coverage manifest's 25 source digests; in the H2 fixture, 12 texts and 5 new keys in fixture.json, the stage contract hash in uk_spine.json, and the synthetic facts the tool sizes to the frames stages 28, 29 and 46 read: property_rental_facts.json (8 values, in their last digits), property_purchase_targets.json (3 values, two by 4 to 5 per cent) and cgt_asset_type_facts.json (28 values, by up to 4.4 per cent). Tests (engine-free, synthetic or code facts): - The helper is the id order on input without ties, is a permutation, equals the full lexsort on tie-heavy input (NaNs and signed zeros included), breaks a tie otherwise than ids would, does not depend on the row order, and refuses float, repeated and misaligned ids. - At each site, a tie across the cut, with ids whose keyed order differs from id order, follows the keyed order computed in the test; distinct values give the id order's result bit for bit; permuting the rows changes nothing. Stage 44's plans and remainder give ties the keyed order reversed, and every _ranked call reads the one keyed draw. - Clone-order guard (verifier P10): with three clones of 60 tied families, clone 0's share of the taken (stages 42 and 26), of the top receipts (stage 28) and of the first ranked (stage 44) lies inside the four-sigma binomial band around a third; under id order it is one (stage 26: the last clone, every time). - The CGT support scenario still selects households 2, 3 and 6; the partway walk test now takes the tied clone the keyed order puts first. The split's walk and its rerun pass the declared pair. The identity tool stays green on a tie at the crossing at three clones, passes the declared pair, and reports the family as missing and clone 0's as extra when its recompute walks by id. The seed and salt pins in test_uk_source_stages and test_uk_frs_spine follow. - 12 mutations of a scratch copy are each caught and the copy as is passes: id order in the helper and in _ranked, keying on the geographic-branch-stripped id (spine_household_keys) in either, the rank-inside-a-tie form at stages 26 and 28, the rerun on another seed, a tie salt equal to a finance-cost salt, salts without the period, the asset type tie held at seed 555 whatever the BADR seed, and the declared seeds without the splits. - The 11 touched test files and test_uk_graph pass (694 tests), as do 8 related files (357: the coverage manifest, spec bundle, country spec, copy-id table, rules-engine provenance and fixture-reading tests). Ruff is clean on the 19 edited Python files and ci_test_plan verify is ok. source_stages.json and the coverage manifest are current, the H2 fixture regenerates and passes its check, and H2 parity passes through the engine slot. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-keyed-tie-breaks.changed.md | 12 + .../spec_engine/schema/sources.schema.json | 23 ++ .../microcosm/build/stochastic_assignment.py | 63 +++++ .../uk/release_input_coverage_manifest.json | 50 ++-- .../src/microcosm/build/uk/source_stages.json | 35 ++- .../src/microcosm/build/uk/spec/sources.yaml | 29 +- .../build/uk_runtime/cgt_asset_type.py | 70 +++-- .../build/uk_runtime/cgt_imputation.py | 98 +++++-- .../microcosm/build/uk_runtime/cgt_support.py | 103 +++++-- .../uk_runtime/high_value_home_support.py | 74 +++-- .../build/uk_runtime/property_components.py | 59 +++- .../microcosm/build/uk_runtime/spine_build.py | 14 + .../shared/test_stochastic_assignment.py | 87 ++++++ .../engine_free/uk/test_uk_cgt_asset_type.py | 106 ++++++- .../engine_free/uk/test_uk_cgt_imputation.py | 232 +++++++++++++++- .../uk/test_uk_cgt_source_manifest.py | 12 + .../engine_free/uk/test_uk_cgt_support.py | 258 ++++++++++++++++-- .../tests/engine_free/uk/test_uk_frs_spine.py | 8 +- .../uk/test_uk_high_value_home_support.py | 176 ++++++++++++ .../uk/test_uk_identity_stability_receipts.py | 43 +++ .../uk/test_uk_property_components.py | 178 ++++++++++++ .../uk/test_uk_property_localisation.py | 2 + .../engine_free/uk/test_uk_source_stages.py | 19 +- .../sources/cgt_asset_type_facts.json | 56 ++-- .../parity/uk_spine/sources/fixture.json | 29 +- .../sources/property_purchase_targets.json | 6 +- .../sources/property_rental_facts.json | 16 +- .../fixtures/parity/uk_spine/uk_spine.json | 2 +- tools/verify_uk_identity_stability.py | 16 +- 29 files changed, 1641 insertions(+), 235 deletions(-) create mode 100644 changelog.d/931-keyed-tie-breaks.changed.md diff --git a/changelog.d/931-keyed-tie-breaks.changed.md b/changelog.d/931-keyed-tie-breaks.changed.md new file mode 100644 index 0000000000..1ef51f4682 --- /dev/null +++ b/changelog.d/931-keyed-tie-breaks.changed.md @@ -0,0 +1,12 @@ +Six walks on the UK spine that ordered exactly tied rows by id now order them by an identity-keyed uniform, then by id (#931; accepted as designed on 7 Nov). Ties broken by id alone put the lowest id first every time, and the copies and geographic clones of one household differ only in their copy ordinal, so the first copy or clone won every tie: on synthetic clone pairs the lower clone index ranked first in every tied pair, and in about half of them once ties are keyed. Values move only where an exact tie touches a cut or a position schedule. + +- **Shared helper.** `microcosm.build.stochastic_assignment.keyed_tie_order(primary, ids, *, seed, salt)` returns `np.lexsort((ids, u, primary))`, with `u` the records' `keyed_tie_uniforms(ids, *, seed, salt)`: `stable_identity_uniforms` of `integer_record_keys(ids, label=salt)`. It is a secondary sort key, the form stage 27 already uses, so on distinct values the order is the id order's bit for bit; it draws the uniforms of tied records only. Each record is keyed on its own id, never on `spine_household_keys`, which strips the geographic branch and would tie the clones again, and the rank-inside-a-tie form is not used, since it moves records that have no tie off their cut. +- **Stage 42** (`cgt_support_split`): households tied on investable wealth, seed 0, salt `cgt_support_split:wealth_tie`. High-value families tie exactly on the four wealth columns, since no stage from 26 to 42 writes them. +- **Stage 26** (`high_value_home_support`): owners tied on home value before the weighted mid-rank, which is unchanged; seed 0, salt `high_value_home_support:value_tie`. +- **Stage 28** (`property_components`): landlords tied on profit in the receipts walk, under the walk seed and a new `tie_salt`, `allocate_property_rental_income:tie`. The stage refuses an empty tie salt and one equal to a salt its finance-cost draws key the same person ids on. The walk's one offset stays as it is (ruling 6). +- **Stage 44** (`hmrc_cgt_gains_spine`): one uniform per person under the stage's seed (552), salted `hmrc_cgt_gains_spine:rank_tie:`, drawn once per stage call and read by every cell walk, pooled walk, plan and the sub-AEA remainder, which take it as `_ranked`'s keyword-only `tie` (without it, person_id alone breaks ties, as before). The plans and the remainder walk the rank ascending, so a tie's member with the higher uniform takes the lower stratum. +- **Stage 47** (`hmrc_cgt_asset_type_spine`): the BADR walk and the main asset type walk, under the BADR seed (555), salted `assign_badr_qualifying_gains:tie:` and `assign_main_asset_type:tie:`. +- **Declarations.** The two support splits declare their tie-break as new `seed` and `salt` keys and the receipts walk its `tie_salt`, in the code dictionaries, the strict schema (three operations), `sources.yaml` and `source_stages.json`; the selection, stratum rule, order, ordering, mapping, realization and values texts, the stage notes and the receipts walk's and asset types' receipt order texts say the above, and "no draw, no seed, no salt" and "deterministic, no seed" are gone. The build sidecar's declared seeds register the splits under their salts and the receipts walk's seed under its tie salt. +- **Stage 42's rule B.** The split's walk, `keeps_the_split`'s rerun of it and the identity tool's recompute all pass the declared pair (`cgt_support_tie_break()`); the selection takes no default, so neither the rerun nor the tool can walk ties in another order. The exact-total correction's candidate order (the heaviest households left whole first, tied weights in row order) is unchanged: it picks the row of a rounding-sized correction rather than drawing anything, and `copy_rule` states it. +- **No nesting promised.** All six walks are pooled. The uniform is a function of each row's own id, so clone 0's equals the `K = 1` row's and every clone draws its own. +- **What moves.** Nothing where no exact tie touches a cut or a position schedule. On synthetic frames (counts only): at stage 42, a band's crossing household when it sits in a tie group, so with three clones per household 2 to 8 of about 850 households per frame changed copy count, and clone 0, which carried the most families in all six frames, carries the most in one; at one clone none moved, since the tied families' members there weigh too little to be copied; at stage 26, the tie group across a region's cut, 0 to 5 of 3,000 owners per frame; at stage 28, every landlord in a tie group can take a tied neighbour's receipts, about two thirds of the tied landlords' receipts moved and none outside the tie groups, the band counts unchanged; at stage 44, often, since the blended rank key ties gainers even on distinct inputs, so 43 to 99 of about 1,800 gainers moved amount where wealth varies and none where it does not; at stage 47, nothing, since plan amounts do not tie (three frames bit for bit). Every spine node re-keys, since `stochastic_assignment` changed and the spine's root binds it; values move from stage 26 on. `source_stages.json`, the coverage manifest's 25 source digests, the H2 fixture's declarations and stage contract hash, and the 39 values of the synthetic facts its tool sizes to the frames stages 28, 29 and 46 read are regenerated. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 4b5e283890..4f581481ed 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -10333,6 +10333,8 @@ "wealth_measure", "investable_wealth_columns", "selection", + "seed", + "salt", "maximum_copy_weight", "copy_rule", "id_remapping", @@ -10424,6 +10426,13 @@ "selection": { "type": "string" }, + "seed": { + "type": "integer" + }, + "salt": { + "type": "string", + "minLength": 1 + }, "maximum_copy_weight": { "type": "number" }, @@ -10467,6 +10476,8 @@ "stratum_lower", "copies", "stratum_rule", + "seed", + "salt", "copy_rule", "id_remapping", "flag_column", @@ -10494,6 +10505,13 @@ "stratum_rule": { "type": "string" }, + "seed": { + "type": "integer" + }, + "salt": { + "type": "string", + "minLength": 1 + }, "copy_rule": { "type": "string" }, @@ -12997,6 +13015,7 @@ "order", "walk", "walk_seed", + "tie_salt", "within_band", "top_band", "maximum_top_band_multiplier", @@ -13034,6 +13053,10 @@ "walk_seed": { "type": "integer" }, + "tie_salt": { + "type": "string", + "minLength": 1 + }, "within_band": { "type": "string" }, diff --git a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py index c72635e69c..5be91c2d30 100644 --- a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py +++ b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py @@ -16,6 +16,8 @@ "binomial_share_tolerance", "clipped_normal_from_uniforms", "integer_record_keys", + "keyed_tie_order", + "keyed_tie_uniforms", "sample_categorical_from_counts", "stable_identity_uniform", "stable_identity_uniforms", @@ -145,6 +147,67 @@ def integer_record_keys( return keys +def keyed_tie_uniforms( + ids: Sequence[object] | np.ndarray, *, seed: int, salt: str +) -> np.ndarray: + """The uniforms :func:`keyed_tie_order` breaks exact ties on, one per id. + + ``stable_identity_uniforms`` of ``ids`` checked by + :func:`integer_record_keys`, so each record's uniform depends on its own + id alone, in whichever pool it is walked. A stage that orders the same + records in several walks draws them once here and sorts on them, as + :func:`keyed_tie_order` does (microcosm#931). + """ + + keys = integer_record_keys(ids, label=salt) + return stable_identity_uniforms(keys, seed=seed, salt=salt) + + +def keyed_tie_order( + primary: Sequence[float] | np.ndarray, + ids: Sequence[object] | np.ndarray, + *, + seed: int, + salt: str, +) -> np.ndarray: + """Positions that sort ``primary`` ascending, exact ties in keyed order. + + The order is ``np.lexsort((ids, u, primary))``, ``u`` the records' + :func:`keyed_tie_uniforms`. Records whose ``primary`` values differ keep + the order those values give them, so on distinct values it is + ``np.lexsort((ids, primary))`` bit for bit; records tied exactly on + ``primary`` take the order of their uniforms, then of their ids. Ties + broken on ids alone put the lowest id first every time, so where the + copies or geographic clones of one household tie, the lowest copy + ordinal always won (microcosm#931). Each uniform is a function of its + record's id, so the order does not depend on the order of the rows. Pass + a negated key to walk it descending. + + Only the records that share their ``primary`` value with another are + drawn, since no other record's uniform can move it; NaNs tie with each + other, as the sort puts them together. + """ + + values = np.asarray(primary, dtype=np.float64) + keys = integer_record_keys(ids, label=salt) + if values.shape != keys.shape: + raise ValueError("primary and ids must be aligned 1-D arrays.") + order = np.lexsort((keys, values)) + ordered = values[order] + same = (ordered[1:] == ordered[:-1]) | ( + np.isnan(ordered[1:]) & np.isnan(ordered[:-1]) + ) + if not same.any(): + return order + tied = np.zeros(order.size, dtype=bool) + tied[1:] |= same + tied[:-1] |= same + rows = order[tied] + uniforms = np.zeros(order.size, dtype=np.float64) + uniforms[rows] = stable_identity_uniforms(keys[rows], seed=seed, salt=salt) + return np.lexsort((keys, uniforms, values)) + + def binomial_share_tolerance(*, target: float, rows: int, sigma: float) -> float: """Binomial band for a share realized over ``rows`` identity-keyed draws. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 2ae9749a85..cacff20e6c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -234,7 +234,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +255,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +279,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +369,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +396,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +468,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +494,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,7 +535,7 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" @@ -558,7 +558,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +579,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +602,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +626,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +654,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +687,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +710,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +731,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +774,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +813,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "2a900684b953332f9a9a2876252c9904902c5ce113c493a52fd26730cbac6ad0", + "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index c1fd6260ad..d50205be04 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3907,17 +3907,19 @@ "kind": "split_top_value_owner_households", "stratum_lower": 0.95, "copies": 6, - "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", + "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", "copy_rule": "every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", "flag_column": "household_is_high_value_support_copy", "copies_column": "high_value_support_copies", "copy_index_column": "high_value_support_copy_index", - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": "every column copied unchanged; the stratum's tie-break is the stage's only draw", "weight_kind_out": "importance", "conservation": "exact_total", "declared_factor": 1.0, - "reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved." + "reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", + "salt": "high_value_home_support:value_tie", + "seed": 0 } ], "outputs": [ @@ -3926,7 +3928,7 @@ "high_value_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made here." + "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned here; the one draw is the stratum's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders owners tied on value, so no copy or clone ordinal ranks highest on every tie." }, { "stage": "regional_home_values", @@ -4092,14 +4094,15 @@ 100000 ], "band_basis": "PRIS 2024-25 Table 13 (all tax entities) scaled by Table 1's individuals' share of landlords", - "order": "profit descending, person_id ascending", + "order": "profit descending, landlords tied on profit in ascending order of a uniform keyed on person_id under walk_seed and tie_salt, then person_id ascending", "walk": "top band first, each band filled with whole persons to its share of the individuals-basis landlords times the spine's landlord weight (the spine's total before calibration differs from PRIS's, and filling the published counts would push every profit rank into a higher band), at one seeded systematic offset (the CGT Table 3 walk's rule); the lowest band takes every remaining landlord", "walk_seed": 0, "within_band": "linear in the weight-ranked position within the band, from its upper bound for the highest profit to its lower bound; never below the landlord's profit", "top_band": "max(100,000, k x profit) with k at least 1, solved by bisection so the band's weighted mean receipts equal the individuals' Table 2 receipts less the bounded bands' realised mean receipts at their individuals-basis counts, over the top band's count; where the floor at k = 1 already exceeds that mean the multiplier stays 1 and the overshoot per landlord is recorded", "maximum_top_band_multiplier": 5.0, "sublet_receipts": "the household reference person's sub-letting rent adds to receipts with no expenses; Rent a Room receipts, which the engine's receipts concept excludes, until PolicyEngine/policyengine-uk#2002 is released", - "residuals": "landlords without profit (FRS loss-makers, whose loss counts as zero, and tape landlords at zero) carry no receipts although PRIS counts them; property_finance_costs_brought_forward stays unset" + "residuals": "landlords without profit (FRS loss-makers, whose loss counts as zero, and tape landlords at zero) carry no receipts although PRIS counts them; property_finance_costs_brought_forward stays unset", + "tie_salt": "allocate_property_rental_income:tie" }, { "kind": "record_mass_conservation_receipt", @@ -7401,18 +7404,20 @@ "other_residential_property_value", "non_residential_property_value" ], - "selection": "within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", + "selection": "within each income band walk households in investable wealth descending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "maximum_copy_weight": 60.0, "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make", "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "flag_column": "household_is_cgt_support_copy", "copies_column": "cgt_support_copies", "copy_index_column": "cgt_support_copy_index", - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": "every column copied unchanged; the selection's tie-break is the stage's only draw", "weight_kind_out": "importance", "conservation": "exact_total", "declared_factor": 1.0, - "reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved." + "reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", + "salt": "cgt_support_split:wealth_tie", + "seed": 0 } ], "outputs": [ @@ -7421,7 +7426,7 @@ "cgt_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." }, { "stage": "cgt_incidence_clone", @@ -7616,7 +7621,7 @@ { "kind": "rank_preserving_allocation", "within": "income band x age group x region group cell, then the income band's pooled unassigned gainers", - "ordering": "a rank key descending, person_id ascending on ties: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers", + "ordering": "a rank key descending, then the rank-tie uniform ascending, then person_id ascending: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers; the rank-tie uniform is one identity-keyed uniform per person, keyed on person_id under seed_base and salted hmrc_cgt_gains_spine:rank_tie:, which every walk, plan and the remainder read", "wealth_rank_weight": 0.75, "investable_wealth_columns": [ "gross_financial_wealth", @@ -7655,8 +7660,8 @@ "resource_vintage": "2017-18 nominal pounds as published (Advani and Summers 2020, CAGE WP 465, Table A1); no uprating is applied", "banding": "Advani-Summers total-income bands at the person's component-sum income without allowance subtraction, the banding the prior draw uses", "range": "quantiles strictly between the band's degree-1 quantile spline zero crossing and its crossing of the build period's annual exempt amount, so every amount lies in (0, AEA]", - "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping draws nothing", - "ordering": "existing gains ascending, person_id descending on ties (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key)", + "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping makes no draw of its own", + "ordering": "existing gains ascending, then the rank-tie uniform descending, then person_id descending (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key)", "quantile_points": [ 0.05, 0.1, @@ -7926,7 +7931,7 @@ "qualifying_amount": "below the lifetime limit a claimant's whole net gain qualifies; in the open top band the qualifying gain is the lifetime limit", "top_band_count": "the published top-band qualifying gains divided by the lifetime limit; the published count is rounded to the nearest thousand", "basis_assumption": "Table 4.1 counts qualifying gains before losses and the annual exempt amount while the frame carries net gains after losses; Investors' Relief is merged with BADR, as in the engine's capital_gains_badr input; the whole-gain assumption reproduces Table 4.1's tax column to within half a percent", - "realization": "weighted systematic sampling within each band in ascending (gain, person_id) order with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare", + "realization": "weighted systematic sampling within each band in ascending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under seed and salted assign_badr_qualifying_gains:tie:, then person_id ascending, with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare", "seed": 555, "seed_mixing": "seed combined with the build period", "policy_parameters": [ @@ -7969,7 +7974,7 @@ "weight_fitting": "one fit per claimant status: multiplicative updates of the type weights until the household-weighted expected gains shares by type match that status's share targets; a fit refuses if it has not converged within the iteration limit, and the stage refuses up front when claimants hold more of the gains than the claimant categories' share", "weight_fitting_iterations": 200, "weight_fitting_tolerance": 1e-06, - "realization": "error diffusion per claimant status in descending (gain, person_id) order: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; deterministic, no seed", + "realization": "error diffusion per claimant status in descending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under the assign_badr_qualifying_gains seed and salted assign_main_asset_type:tie:, then person_id ascending: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; that tie-break is its only draw", "output_column": "capital_gains_asset_type", "value_domain": [ "none", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 48cc7f8eab..9d6766d069 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3328,13 +3328,15 @@ stages: - kind: split_top_value_owner_households stratum_lower: 0.95 copies: 6 - stratum_rule: within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected + stratum_rule: within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected + seed: 0 + salt: high_value_home_support:value_tie copy_rule: every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight id_remapping: copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12) flag_column: household_is_high_value_support_copy copies_column: high_value_support_copies copy_index_column: high_value_support_copy_index - values: every column copied unchanged; no draw, no seed, no salt + values: every column copied unchanged; the stratum's tie-break is the stage's only draw weight_kind_out: importance conservation: exact_total declared_factor: 1.0 @@ -3344,7 +3346,7 @@ stages: - high_value_support_copies - high_value_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split for the dense graph''s post-geography home value map (microcosm#1240). The value node reads every owner clone''s home value off its assigned constituency''s Land Registry sale distribution at the household''s weighted rank among its region''s owners, so each seat''s homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made here.' + notes: 'Spine-only, deterministic, mass-conserving support split for the dense graph''s post-geography home value map (microcosm#1240). The value node reads every owner clone''s home value off its assigned constituency''s Land Registry sale distribution at the household''s weighted rank among its region''s owners, so each seat''s homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned here; the one draw is the stratum''s tie-break, a uniform keyed on each household''s own household_id (seed, salt) that orders owners tied on value, so no copy or clone ordinal ranks highest on every tie.' - stage: regional_home_values survey: HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level) source: uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations @@ -3468,9 +3470,10 @@ stages: population: persons with profit from property other than sub-letting above zero, on both channels, after the finance-cost add-back band_lower_bounds: [0, 10000, 20000, 30000, 40000, 50000, 60000, 70000, 80000, 90000, 100000] band_basis: PRIS 2024-25 Table 13 (all tax entities) scaled by Table 1's individuals' share of landlords - order: profit descending, person_id ascending + order: profit descending, landlords tied on profit in ascending order of a uniform keyed on person_id under walk_seed and tie_salt, then person_id ascending walk: top band first, each band filled with whole persons to its share of the individuals-basis landlords times the spine's landlord weight (the spine's total before calibration differs from PRIS's, and filling the published counts would push every profit rank into a higher band), at one seeded systematic offset (the CGT Table 3 walk's rule); the lowest band takes every remaining landlord walk_seed: 0 + tie_salt: allocate_property_rental_income:tie within_band: linear in the weight-ranked position within the band, from its upper bound for the highest profit to its lower bound; never below the landlord's profit top_band: max(100,000, k x profit) with k at least 1, solved by bisection so the band's weighted mean receipts equal the individuals' Table 2 receipts less the bounded bands' realised mean receipts at their individuals-basis counts, over the top band's count; where the floor at k = 1 already exceeds that mean the multiplier stays 1 and the overshoot per landlord is recorded maximum_top_band_multiplier: 5.0 @@ -6013,14 +6016,16 @@ stages: - corporate_wealth - other_residential_property_value - non_residential_property_value - selection: within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused + selection: within each income band walk households in investable wealth descending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused + seed: 0 + salt: cgt_support_split:wealth_tie maximum_copy_weight: 60.0 copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make id_remapping: copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused flag_column: household_is_cgt_support_copy copies_column: cgt_support_copies copy_index_column: cgt_support_copy_index - values: every column copied unchanged; no draw, no seed, no salt + values: every column copied unchanged; the selection's tie-break is the stage's only draw weight_kind_out: importance conservation: exact_total declared_factor: 1.0 @@ -6030,7 +6035,7 @@ stages: - cgt_support_copies - cgt_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool''s selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' + notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool''s selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk''s tie-break, a uniform keyed on each household''s own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' - stage: cgt_incidence_clone survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 @@ -6189,7 +6194,7 @@ stages: reporting_grain: Table 6 age bands and the twelve region-tier areas - kind: rank_preserving_allocation within: income band x age group x region group cell, then the income band's pooled unassigned gainers - ordering: 'a rank key descending, person_id ascending on ties: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers' + ordering: 'a rank key descending, then the rank-tie uniform ascending, then person_id ascending: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers; the rank-tie uniform is one identity-keyed uniform per person, keyed on person_id under seed_base and salted hmrc_cgt_gains_spine:rank_tie:, which every walk, plan and the remainder read' wealth_rank_weight: 0.75 investable_wealth_columns: - gross_financial_wealth @@ -6223,8 +6228,8 @@ stages: resource_vintage: 2017-18 nominal pounds as published (Advani and Summers 2020, CAGE WP 465, Table A1); no uprating is applied banding: Advani-Summers total-income bands at the person's component-sum income without allowance subtraction, the banding the prior draw uses range: quantiles strictly between the band's degree-1 quantile spline zero crossing and its crossing of the build period's annual exempt amount, so every amount lies in (0, AEA] - mapping: remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping draws nothing - ordering: existing gains ascending, person_id descending on ties (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key) + mapping: remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping makes no draw of its own + ordering: existing gains ascending, then the rank-tie uniform descending, then person_id descending (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key) quantile_points: - 0.05 - 0.1 @@ -6440,7 +6445,7 @@ stages: qualifying_amount: below the lifetime limit a claimant's whole net gain qualifies; in the open top band the qualifying gain is the lifetime limit top_band_count: the published top-band qualifying gains divided by the lifetime limit; the published count is rounded to the nearest thousand basis_assumption: Table 4.1 counts qualifying gains before losses and the annual exempt amount while the frame carries net gains after losses; Investors' Relief is merged with BADR, as in the engine's capital_gains_badr input; the whole-gain assumption reproduces Table 4.1's tax column to within half a percent - realization: weighted systematic sampling within each band in ascending (gain, person_id) order with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare + realization: weighted systematic sampling within each band in ascending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under seed and salted assign_badr_qualifying_gains:tie:, then person_id ascending, with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare seed: 555 seed_mixing: seed combined with the build period policy_parameters: @@ -6477,7 +6482,7 @@ stages: weight_fitting: 'one fit per claimant status: multiplicative updates of the type weights until the household-weighted expected gains shares by type match that status''s share targets; a fit refuses if it has not converged within the iteration limit, and the stage refuses up front when claimants hold more of the gains than the claimant categories'' share' weight_fitting_iterations: 200 weight_fitting_tolerance: 1.0e-06 - realization: 'error diffusion per claimant status in descending (gain, person_id) order: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; deterministic, no seed' + realization: 'error diffusion per claimant status in descending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under the assign_badr_qualifying_gains seed and salted assign_main_asset_type:tie:, then person_id ascending: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; that tie-break is its only draw' output_column: capital_gains_asset_type value_domain: - none diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py index 93fb2dae7b..cd55e7650e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py @@ -106,7 +106,10 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.stochastic_assignment import stable_identity_uniform +from microcosm.build.stochastic_assignment import ( + keyed_tie_order, + stable_identity_uniform, +) from microcosm.build.uk_runtime.cgt_imputation import ( UKCGTPolicyParameters, uk_cgt_policy_parameters, @@ -258,8 +261,10 @@ #: The BADR seed is combined with the build period, as the amounts stage #: does; distinct from the amounts stage's 552 so the two stages never share -#: a draw. The residential flag draws nothing since microcosm#1063: the -#: ``cgt_residential_split`` stage carries its probability as weight. +#: a draw. It also keys the tie-breaks of the BADR walk and the main asset +#: type walk, each under its own salt (microcosm#931). The residential flag +#: draws nothing since microcosm#1063: the ``cgt_residential_split`` stage +#: carries its probability as weight. CGT_BADR_FLAG_SEED = 555 _BISECTION_ITERATIONS = 200 @@ -1042,6 +1047,9 @@ def _assign_badr_claims( one receipt row per band. Each band's walk offset is an identity-keyed uniform on its bounds (``_badr_band_offset``), so it depends neither on which bands carry a target nor on which other bands the facts declare. + The walk runs in ascending gain order, persons tied on gain in the order + of a uniform keyed on their person_id, salted with the build period, then + of their ids (``keyed_tie_order``; microcosm#931). """ qualifying = np.zeros(gains.shape) @@ -1125,7 +1133,12 @@ def _assign_badr_claims( np.log(band_gains) - centre, intercept, slope, band_offset ) amounts = band_gains - order = np.lexsort((person_id[rows], band_gains)) + order = keyed_tie_order( + band_gains, + person_id[rows], + seed=seed, + salt=f"assign_badr_qualifying_gains:tie:{int(time_period)}", + ) flags = _weighted_systematic_flags( probabilities, band_weights, @@ -1546,8 +1559,15 @@ def assign_uk_cgt_asset_types( ) stakes = person_weight[rows] * gains[rows] # Largest gain first, so the rows that could move the composition most - # are placed while every later row can still repay the balance. - order = np.lexsort((person_id[rows], -gains[rows])) + # are placed while every later row can still repay the balance; persons + # tied on gain take the order of a uniform keyed on their person_id, + # then of their ids (microcosm#931). + order = keyed_tie_order( + -gains[rows], + person_id[rows], + seed=badr_seed, + salt=f"assign_main_asset_type:tie:{int(time_period)}", + ) choice, balances = _diffuse_types(type_probabilities, stakes, order) remainder_types[in_status] = choice mass = float(stakes.sum()) @@ -1679,7 +1699,10 @@ def assign_uk_cgt_asset_types( "log_sigma": CGT_ASSET_TYPE_LOG_SIGMA, "median_anchor_gbp": medians, "composition_by_claimant_status": status_reports, - "realization": "error diffusion in descending (gain, person_id) order", + "realization": ( + "error diffusion in descending gain order, ties in ascending order of " + "a uniform keyed on person_id, then person_id ascending" + ), "achieved_median_gain_gbp": { asset_type_name: _weighted_median( gains[asset_type == asset_type_name], @@ -1949,13 +1972,16 @@ def cgt_asset_type_operation_parameters() -> dict[str, dict[str, Any]]: "Table 4.1's tax column to within half a percent" ), "realization": ( - "weighted systematic sampling within each band in ascending " - "(gain, person_id) order with one identity-keyed offset per " - "band, keyed on its bounds (|, open for the top band) and salted " - "assign_badr_qualifying_gains:offset:, with no stream " - "order, so a band's offset depends neither on which bands carry " - "a target nor on which other bands the facts declare" + "weighted systematic sampling within each band in ascending gain " + "order, persons tied on gain in ascending order of a uniform " + "keyed on person_id under seed and salted " + "assign_badr_qualifying_gains:tie:, then person_id " + "ascending, with one identity-keyed offset per band, keyed on " + "its bounds (|, open for the " + "top band) and salted assign_badr_qualifying_gains:offset:, " + "with no stream order, so a band's offset depends neither on " + "which bands carry a target nor on which other bands the facts " + "declare" ), "seed": CGT_BADR_FLAG_SEED, "seed_mixing": "seed combined with the build period", @@ -2025,13 +2051,15 @@ def cgt_asset_type_operation_parameters() -> dict[str, dict[str, Any]]: "weight_fitting_iterations": _SHARE_FIT_ITERATIONS, "weight_fitting_tolerance": _SHARE_FIT_TOLERANCE, "realization": ( - "error diffusion per claimant status in descending (gain, " - "person_id) order: each person adds weight x gain x type " - "probability to a running balance per type and takes the type it " - "may hold with the largest balance, which is drawn down by the " - "whole weight x gain, so the realised gains by type track the " - "fitted composition instead of following the heaviest rows; " - "deterministic, no seed" + "error diffusion per claimant status in descending gain order, " + "persons tied on gain in ascending order of a uniform keyed on " + "person_id under the assign_badr_qualifying_gains seed and salted " + "assign_main_asset_type:tie:, then person_id ascending: " + "each person adds weight x gain x type probability to a running " + "balance per type and takes the type it may hold with the largest " + "balance, which is drawn down by the whole weight x gain, so the " + "realised gains by type track the fitted composition instead of " + "following the heaviest rows; that tie-break is its only draw" ), "output_column": CGT_ASSET_TYPE_COLUMN, "value_domain": list(CGT_ASSET_TYPE_DOMAIN), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index 4c2b2b1f1a..a5fad7b30c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -99,7 +99,10 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.stochastic_assignment import stable_identity_uniform +from microcosm.build.stochastic_assignment import ( + keyed_tie_uniforms, + stable_identity_uniform, +) from microcosm.build.uk_runtime.advani_summers import ( ADVANI_SUMMERS_RESOURCE, ADVANI_SUMMERS_VINTAGE, @@ -170,13 +173,14 @@ UK_CGT_SPINE_STAGE_NAME = "hmrc_cgt_gains_spine" #: Base seed for the stage's draws. Combined with the build period, which -#: salts every walk offset and plan jitter, so two periods draw differently -#: while each build is reproducible. +#: salts every walk offset, plan jitter and rank-tie uniform, so two periods +#: draw differently while each build is reproducible. UK_CGT_IMPUTATION_SEED = 552 #: How the sub-AEA remainder takes its amounts (microcosm#970): the #: Advani-Summers within-band quantile function restricted to the exempt -#: range, rank-preserving within each total-income band, no seeded draw. +#: range, rank-preserving within each total-income band, no seeded draw of +#: its own (ties on the prior gain take the allocation's rank-tie uniforms). UK_CGT_REMAINDER_POLICY = "advani_summers_within_band_exempt_range" #: The reviewed mass-conservation receipt the spine stage records. The @@ -1294,6 +1298,7 @@ def _draw_plan_amounts( weights: np.ndarray, rng: np.random.Generator | _KeyedOffset, new_gains: np.ndarray, + tie: np.ndarray | None = None, ) -> None: """Draw one plan's amounts on weight-proportional quantile strata. @@ -1307,12 +1312,13 @@ def _draw_plan_amounts( third of whose weight sat in the bottom strata). The open band takes each stratum's conditional mean, so its weighted mean is the published mean exactly; a bounded band takes the quantile inside each stratum, which - reproduces the mean to the width of the widest stratum. + reproduces the mean to the width of the widest stratum. ``tie`` orders + members tied on the key as :func:`_ranked` does, reversed. """ if members.size == 0: return - ordered = _ranked(members, person_id=person_id, existing=existing)[::-1] + ordered = _ranked(members, person_id=person_id, existing=existing, tie=tie)[::-1] cumulative = np.cumsum(weights[ordered]) total = float(cumulative[-1]) if not total > 0.0: @@ -1336,12 +1342,27 @@ def _draw_plan_amounts( def _ranked( - indices: np.ndarray, *, person_id: np.ndarray, existing: np.ndarray + indices: np.ndarray, + *, + person_id: np.ndarray, + existing: np.ndarray, + tie: np.ndarray | None = None, ) -> np.ndarray: - # Rank by the given key, largest first: the allocation's wealth-blended - # rank key, or the prior gain for the sub-AEA remainder. person_id breaks - # ties so the ordering, and with it every draw, is deterministic. - order = np.lexsort((person_id[indices], -existing[indices])) + """``indices`` by the given key, largest first. + + The key is the allocation's wealth-blended rank key, or the prior gain + for the sub-AEA remainder. Persons tied on it take the order of ``tie`` + ascending, one identity-keyed uniform per person + (:func:`microcosm.build.stochastic_assignment.keyed_tie_uniforms`, the + uniforms its ``keyed_tie_order`` sorts on), then of person_id, so no id + range, the copies or clones of one household among them, wins every tie + (microcosm#931). With no ``tie``, person_id alone breaks ties. + """ + + if tie is None: + order = np.lexsort((person_id[indices], -existing[indices])) + else: + order = np.lexsort((person_id[indices], tie[indices], -existing[indices])) return indices[order] @@ -1355,6 +1376,7 @@ def _map_remainder_amounts( rows: Sequence[Mapping[str, object]], annual_exempt_amount: float, new_gains: np.ndarray, + tie: np.ndarray | None = None, ) -> dict[str, object]: """Place the sub-AEA remainder on the A&S within-band surface (microcosm#970). @@ -1367,14 +1389,17 @@ def _map_remainder_amounts( its crossing of the annual exempt amount, so the band's weighted remainder reproduces the published conditional shape on (0, AEA]. The mapping is a monotone transform of the existing gains inside each band, - consumes no seeded draw, and fails closed if any amount leaves the - range. Returns the receipt for the allocation report. + makes no draw of its own (``tie``, the allocation's rank-tie uniforms, + orders persons tied on their prior gain, as :func:`_ranked` reversed), + and fails closed if any amount leaves the range. Returns the receipt for + the allocation report. """ band_rows: list[dict[str, object]] = [] for band in np.unique(band_index[remainder]): members = remainder[band_index[remainder] == band] - ordered = _ranked(members, person_id=person_id, existing=existing)[::-1] + ranked = _ranked(members, person_id=person_id, existing=existing, tie=tie) + ordered = ranked[::-1] knots = advani_summers_knots(rows[int(band)]) lower_quantile, upper_quantile = exempt_range_quantiles( knots, annual_exempt_amount @@ -1804,6 +1829,16 @@ def impute_uk_capital_gains_with_report( _, age_group, _, region_group = _person_conditioning_cells(person, household) rank_key, investable_wealth = _allocation_rank_key(person, household, existing) + # Persons tied on a rank take the order of a uniform keyed on their own + # person_id, salted with the build period, then of their ids: one draw per + # person for every walk, plan and the remainder, so the same person wins + # a tie throughout and no id range, the copies or clones of one household + # among them, wins every tie (microcosm#931). + rank_tie = keyed_tie_uniforms( + person_id, + seed=seed, + salt=f"{UK_CGT_SPINE_STAGE_NAME}:rank_tie:{int(time_period)}", + ) # Receipts only (microcosm#1045): the walk never reads the row type. support_family, support_column_present = _support_family_rows(person, household) @@ -1875,7 +1910,10 @@ def keyed(salt: str, key: str) -> _KeyedOffset: carry = -deferred continue ranked = _ranked( - np.flatnonzero(in_cell), person_id=person_id, existing=rank_key + np.flatnonzero(in_cell), + person_id=person_id, + existing=rank_key, + tie=rank_tie, ) weights = person_weight[ranked] # When the cell holds less gainer mass than its target, @@ -1935,7 +1973,9 @@ def keyed(salt: str, key: str) -> _KeyedOffset: gain_lower: value * net_shortfall / positive_total for gain_lower, value in positive.items() } - ranked = _ranked(np.flatnonzero(pool), person_id=person_id, existing=rank_key) + ranked = _ranked( + np.flatnonzero(pool), person_id=person_id, existing=rank_key, tie=rank_tie + ) weights = person_weight[ranked] scale = min(1.0, float(weights.sum()) / net_shortfall) boundaries = [ @@ -1972,11 +2012,13 @@ def keyed(salt: str, key: str) -> _KeyedOffset: weights=person_weight, rng=keyed(jitter_salt, f"plan:{gain_lower}:{income_lower}"), new_gains=new_gains, + tie=rank_tie, ) # Below the published taxpayer mass: sub-AEA gainers take Advani-Summers # within-band amounts restricted to (0, AEA], rank-preserving within each - # A&S total-income band (microcosm#970). Nothing here draws. + # A&S total-income band (microcosm#970). Nothing here draws; ties on the + # prior gain take the rank-tie uniforms. remainder_receipt = _map_remainder_amounts( remainder=np.flatnonzero(is_gainer & ~assigned), existing=existing, @@ -1988,6 +2030,7 @@ def keyed(salt: str, key: str) -> _KeyedOffset: rows=remainder_rows, annual_exempt_amount=parameters.annual_exempt_amount, new_gains=new_gains, + tie=rank_tie, ) if not np.isfinite(new_gains).all(): @@ -2466,10 +2509,14 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: "band's pooled unassigned gainers" ), "ordering": ( - "a rank key descending, person_id ascending on ties: among " - "gainers, the household investable-wealth rank times the wealth " - "rank weight plus the prior-gain rank times its complement, " - "both as average ranks over the gainers" + "a rank key descending, then the rank-tie uniform ascending, then " + "person_id ascending: among gainers, the household " + "investable-wealth rank times the wealth rank weight plus the " + "prior-gain rank times its complement, both as average ranks " + "over the gainers; the rank-tie uniform is one identity-keyed " + "uniform per person, keyed on person_id under seed_base and " + "salted hmrc_cgt_gains_spine:rank_tie:, which every " + "walk, plan and the remainder read" ), "wealth_rank_weight": UK_CGT_WEALTH_RANK_WEIGHT, "investable_wealth_columns": list(UK_CGT_INVESTABLE_WEALTH_COLUMNS), @@ -2600,12 +2647,13 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: "quantile strata that partition the range, evaluated on the " "band's spline; zero-weight persons take a nominal stratum of " "the band's smallest positive weight and carry no mass; the " - "mapping draws nothing" + "mapping makes no draw of its own" ), "ordering": ( - "existing gains ascending, person_id descending on ties (the " - "prior-gain ranking reversed; the allocation and the within-band " - "draws rank on the wealth-blended key)" + "existing gains ascending, then the rank-tie uniform descending, " + "then person_id descending (the prior-gain ranking reversed; the " + "allocation and the within-band draws rank on the wealth-blended " + "key)" ), "quantile_points": list(CGT_QUANTILE_POINTS), "spline_degree": 1, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index 6079801f5a..541cee915e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -14,11 +14,13 @@ identical copies at equal weight, ``K`` the frame's geographic clone count (1 on a national spine; microcosm#931), each carrying its index ``k`` in ``cgt_support_copy_index`` (0 on roots and unsplit households), the explicit -lineage the geography identity kernel keys on. No value moves, no draw is -made, and every household's mass and the total household mass are conserved -exactly; the incidence clone stage (``cgt_incidence_clone``) then gives each -copy's clone its own identity-keyed prior, and the redraw's wealth-ranked -walk places the light rows where it was already placing heavy ones. +lineage the geography identity kernel keys on. No value moves, and every +household's mass and the total household mass are conserved exactly; the +one draw is the walk's tie-break, a uniform keyed on each household's own id +that orders households tied on wealth (microcosm#931). The incidence clone +stage (``cgt_incidence_clone``) then gives each copy's clone its own +identity-keyed prior, and the redraw's wealth-ranked walk places the light +rows where it was already placing heavy ones. The support masses are read from the vendored ``hmrc_cgt_conditioning_facts.json`` (HMRC Table 3, individuals by size of gain and taxable income) through @@ -35,6 +37,7 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import keyed_tie_order from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_INVESTABLE_WEALTH_COLUMNS, @@ -104,6 +107,12 @@ ) #: The Table 3 bands of gains the support is sized on. CGT_SUPPORT_MINIMUM_GAIN_BAND_LOWER = 250_000 +#: The seed and salt of the selection walk's tie-break: households tied on +#: investable wealth take the order of a uniform keyed on their own +#: household_id, then of their ids, not of their ids alone, which always put +#: a household's lowest copy or clone ordinal first (microcosm#931). +CGT_SUPPORT_TIE_SEED = 0 +CGT_SUPPORT_TIE_SALT = "cgt_support_split:wealth_tie" #: The redraw's income bands, so classification matches its walk. CGT_SUPPORT_INCOME_BAND_LOWER_BOUNDS: tuple[int, ...] = ( HMRC_CGT_INCOME_BAND_LOWER_BOUNDS @@ -128,8 +137,10 @@ #: correction costs a few dozen sums of the frame's weights. At the CGT split #: a selected household left at one copy that can carry it also costs a rerun #: of the selection walk (``keeps_the_split``), so 64 such candidates that the -#: check passes over would cost 64 walks, about 26 s at the 0.41 s one walk -#: takes on a synthetic vector the size of a ``K = 15`` spine. A national +#: check passes over would cost 64 walks, about 30 to 45 s at the 0.5 to 0.7 s +#: one walk takes on a synthetic vector the size of a ``K = 15`` spine, the +#: more of it tied on wealth the slower, since the walk's tie-break draws the +#: tied households' uniforms and checks the ids it keys them on. A national #: frame does not reach that, since its 64 heaviest whole households are #: unselected ones, far heavier than ``60 / K``, so the cap bounds the search #: there to the sums. @@ -403,17 +414,24 @@ def select_cgt_support_households( income_band: Sequence[int] | np.ndarray, wealth: Sequence[float] | np.ndarray, support_mass_by_band: Mapping[int, float], + *, + seed: int, + salt: str, ) -> tuple[np.ndarray, tuple[dict[str, object], ...]]: """Select the wealthiest households of each income band, whole. Within each band of ``support_mass_by_band`` (income band lower bound to support mass, walked in mapping order) the positive-weight households of - that band are ordered by wealth descending then household id ascending, + that band are ordered by wealth descending, households tied on wealth by + a uniform keyed on their own household id under ``seed`` and ``salt``, + then by id (:func:`microcosm.build.stochastic_assignment.keyed_tie_order`), and taken whole until the cumulative weight reaches the support mass; the household that crosses it is taken. A pool whose mass falls short is taken entirely and recorded as exhausted, never refused; zero-weight rows are outside every pool. Returns the selection mask aligned to the - inputs and one receipt row per band. + inputs and one receipt row per band. ``seed`` and ``salt`` have no + default: the stage, its rerun of the walk and the identity tool's + recompute each pass the declared pair (:func:`cgt_support_tie_break`). """ ids = np.asarray(household_ids) @@ -444,7 +462,9 @@ def select_cgt_support_households( ) pool = np.flatnonzero((bands == income_lower) & (values > 0.0)) pool_mass = float(values[pool].sum()) - order = pool[np.lexsort((ids[pool], -wealth_values[pool]))] + order = pool[ + keyed_tie_order(-wealth_values[pool], ids[pool], seed=seed, salt=salt) + ] cumulative = np.cumsum(values[order]) if target <= 0.0: take = 0 @@ -606,7 +626,11 @@ def split_cgt_support_households( ) -> UKCGTSupportSplitResult: """Split the wealthiest households of each income band into light copies. - Deterministic and mass-conserving: no draw, no seed, no salt. Each + Deterministic and mass-conserving. Its one draw is the selection walk's + tie-break: households tied on wealth take the order of a uniform keyed + on their own household id under the declared seed and salt + (:func:`cgt_support_tie_break`), then of their ids, and the walk's rerun + below passes the same pair. Each selected household of weight ``w`` becomes ``n = ceil(K x w / 60)`` copies at ``w / n``, ``K`` the frame's geographic clone count (``clone_count``, which the stage transform reads from the frame), so a @@ -616,7 +640,8 @@ def split_cgt_support_households( accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and - not others, lowest household id first. The root keeps its ids, copy + not others, in their keyed order rather than lowest household id first, + so no clone index is favoured. The root keeps its ids, copy ``k`` takes every entity id plus ``k`` times the declared radix of the copy-id table's ``cgt_support_split`` field (so every incoming id must sit below it, and ``n`` is at most the field's bound of 10,000), records @@ -674,8 +699,15 @@ def split_cgt_support_households( support_by_band = { int(row["income_lower_bound"]): float(row["support_mass"]) for row in mass_rows } + tie_seed, tie_salt = cgt_support_tie_break() selected, selection_rows = select_cgt_support_households( - household_ids, weights, income_band, wealth, support_by_band + household_ids, + weights, + income_band, + wealth, + support_by_band, + seed=tie_seed, + salt=tie_salt, ) copies = np.ones(len(household), dtype="int64") copies[selected] = cgt_support_copy_counts( @@ -740,9 +772,10 @@ def keeps_the_split(row: int, weight: float) -> bool: walked band, so its weight moves neither the walk nor its count of one. A selected household left at one copy can move both, so it carries the correction only where the walk rerun on its corrected - weight selects the same households and that weight still counts one - copy: the rule rerun on the stage's output then divides the same - households the same way, as the identity tool's recompute does. + weight, under the stage's declared tie-break, selects the same + households and that weight still counts one copy: the rule rerun on + the stage's output then divides the same households the same way, as + the identity tool's recompute does. """ if not selected[row]: @@ -755,7 +788,13 @@ def keeps_the_split(row: int, weight: float) -> bool: moved = weights.copy() moved[row] = weight reselected, _ = select_cgt_support_households( - household_ids, moved, income_band, wealth, support_by_band + household_ids, + moved, + income_band, + wealth, + support_by_band, + seed=tie_seed, + salt=tie_salt, ) return bool(np.array_equal(reselected, selected)) @@ -933,11 +972,15 @@ def cgt_support_split_operation_parameters() -> tuple[ "investable_wealth_columns": list(UK_CGT_INVESTABLE_WEALTH_COLUMNS), "selection": ( "within each income band walk households in investable wealth " - "descending, household_id ascending, accumulating positive " - "household weight until the cumulative weight reaches the " - "band's support mass; whole households; an exhausted pool is " - "recorded, not refused" + "descending, households tied on it in ascending order of a " + "uniform keyed on their own household_id under seed and salt, " + "then household_id ascending, accumulating positive household " + "weight until the cumulative weight reaches the band's support " + "mass; whole households; an exhausted pool is recorded, not " + "refused" ), + "seed": CGT_SUPPORT_TIE_SEED, + "salt": CGT_SUPPORT_TIE_SALT, "maximum_copy_weight": CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, "copy_rule": ( "copies = ceil(geographic clone count x household weight / " @@ -955,7 +998,10 @@ def cgt_support_split_operation_parameters() -> tuple[ "flag_column": HOUSEHOLD_IS_CGT_SUPPORT_COPY, "copies_column": CGT_SUPPORT_COPIES_COLUMN, "copy_index_column": CGT_SUPPORT_COPY_INDEX_COLUMN, - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": ( + "every column copied unchanged; the selection's tie-break is " + "the stage's only draw" + ), "weight_kind_out": WeightKind.IMPORTANCE.value, "conservation": "exact_total", "declared_factor": 1.0, @@ -965,6 +1011,19 @@ def cgt_support_split_operation_parameters() -> tuple[ ) +def cgt_support_tie_break() -> tuple[int, str]: + """The declared seed and salt of the selection walk's tie-break. + + Read from the reviewed operation the manifests restate verbatim + (:func:`cgt_support_split_operation_parameters`), so the stage's walk, its + rerun of the walk in ``keeps_the_split`` and the identity tool's + recompute all key their ties on the declared pair (microcosm#931). + """ + + ((_, operation),) = cgt_support_split_operation_parameters() + return int(operation["seed"]), str(operation["salt"]) + + def _assert_cgt_support_split_stage_parameters(stage: SourceStageSpec) -> None: """Bind every stage manifest parameter to reviewed code constants. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py index 7b37aa2b57..9c5d2bd25f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py @@ -14,13 +14,14 @@ copy its own keyed rank inside the stratum instead of the household's single rank, so the copies spread across the tail and across seats. -Deterministic and mass-conserving: no draw, no seed, no salt. Every -household's mass and the total household mass are conserved exactly. The -stratum is selected on the spine's weights with ties broken by household id; -the value node ranks the pool's weights with a keyed tie-break and ranks the -copies beside their root, so the two agree up to ties and up to any weight -change between the spine and the pool, not by construction. No value moves -here. +Deterministic and mass-conserving: every household's mass and the total +household mass are conserved exactly, and the one draw is the stratum's +tie-break, a uniform keyed on each household's own id that orders +households tied on value (microcosm#931). The stratum is selected on the +spine's weights with that tie-break; the value node ranks the pool's weights +with a keyed tie-break of its own and ranks the copies beside their root, so +the two agree up to ties and up to any weight change between the spine and +the pool, not by construction. No value moves here. """ from __future__ import annotations @@ -31,6 +32,7 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import keyed_tie_order from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations from microcosm.build.uk_runtime.cgt_support import ( @@ -60,9 +62,12 @@ "HIGH_VALUE_SUPPORT_OPERATION_KIND", "HIGH_VALUE_SUPPORT_STAGE_NAME", "HIGH_VALUE_SUPPORT_STRATUM_LOWER", + "HIGH_VALUE_SUPPORT_TIE_SALT", + "HIGH_VALUE_SUPPORT_TIE_SEED", "UKHighValueHomeSupportResult", "UKHighValueHomeSupportStageTransform", "high_value_support_operation_parameters", + "high_value_support_tie_break", "select_high_value_support_households", "split_high_value_support_households", ] @@ -79,6 +84,12 @@ HIGH_VALUE_SUPPORT_COPIES = 6 #: The copy-id field copy ``k`` takes ``k`` in (microcosm#931). HIGH_VALUE_SUPPORT_COPY_ID_FIELD = "high_value_support" +#: The seed and salt of the stratum's tie-break: owners tied on value take +#: the order of a uniform keyed on their own household_id, then of their ids, +#: not of their ids alone, which always ranked a household's highest copy or +#: clone ordinal highest (microcosm#931). +HIGH_VALUE_SUPPORT_TIE_SEED = 0 +HIGH_VALUE_SUPPORT_TIE_SALT = "high_value_home_support:value_tie" HIGH_VALUE_SUPPORT_MASS_CHANGE_REASON = ( "High-value home support divides every owner household in the top value " "stratum of its region into light copies at equal weight; every household's " @@ -144,11 +155,15 @@ def high_value_support_operation_parameters() -> tuple[ "stratum_rule": ( "within each FRS region, owner households (tenure owned outright " "or with a mortgage, main_residence_value > 0, positive weight) " - "ordered by main_residence_value ascending then household_id " - "ascending; a household whose weighted mid-rank " - "(cumulative weight before it plus half its own, over the " - "region's owner weight) is at or above stratum_lower is selected" + "ordered by main_residence_value ascending, households tied on " + "it in ascending order of a uniform keyed on their own " + "household_id under seed and salt, then household_id ascending; " + "a household whose weighted mid-rank (cumulative weight before " + "it plus half its own, over the region's owner weight) is at or " + "above stratum_lower is selected" ), + "seed": HIGH_VALUE_SUPPORT_TIE_SEED, + "salt": HIGH_VALUE_SUPPORT_TIE_SALT, "copy_rule": ( "every selected household becomes copies rows at weight / " f"copies, {CGT_SUPPORT_EXACT_TOTAL_EXCEPTION}" @@ -160,7 +175,10 @@ def high_value_support_operation_parameters() -> tuple[ "flag_column": HIGH_VALUE_SUPPORT_COPY_COLUMN, "copies_column": HIGH_VALUE_SUPPORT_COPIES_COLUMN, "copy_index_column": HIGH_VALUE_SUPPORT_INDEX_COLUMN, - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": ( + "every column copied unchanged; the stratum's tie-break is the " + "stage's only draw" + ), "weight_kind_out": "importance", "conservation": "exact_total", "declared_factor": 1.0, @@ -170,6 +188,18 @@ def high_value_support_operation_parameters() -> tuple[ ) +def high_value_support_tie_break() -> tuple[int, str]: + """The declared seed and salt of the stratum's tie-break. + + Read from the reviewed operation the manifests restate verbatim + (:func:`high_value_support_operation_parameters`), so the stage keys its + ties on the declared pair (microcosm#931). + """ + + ((_, operation),) = high_value_support_operation_parameters() + return int(operation["seed"]), str(operation["salt"]) + + def select_high_value_support_households( *, household_ids: np.ndarray, @@ -178,13 +208,20 @@ def select_high_value_support_households( tenure_type: np.ndarray, value: np.ndarray, stratum_lower: float = HIGH_VALUE_SUPPORT_STRATUM_LOWER, + seed: int, + salt: str, ) -> tuple[np.ndarray, tuple[dict[str, object], ...]]: """The owner households in the top value stratum of each region. - Returns the selection mask aligned to the inputs and one receipt row per - region (owner households, owner mass, households selected, selected mass - and the value rank the stratum starts at). A region without owner - households selects nothing and says so. + Owners are ranked by value ascending, owners tied on value by a uniform + keyed on their own household id under ``seed`` and ``salt``, then by id + (:func:`microcosm.build.stochastic_assignment.keyed_tie_order`); ``seed`` + and ``salt`` have no default (the stage passes + :func:`high_value_support_tie_break`). Returns the selection mask aligned + to the inputs and one receipt row per region (owner households, owner + mass, households selected, selected mass and the value rank the stratum + starts at). A region without owner households selects nothing and says + so. """ ids = np.asarray(household_ids) weight = np.asarray(weights, dtype=float) @@ -216,7 +253,7 @@ def select_high_value_support_households( } ) continue - order = pool[np.lexsort((ids[pool], values[pool]))] + order = pool[keyed_tie_order(values[pool], ids[pool], seed=seed, salt=salt)] pool_weight = weight[order] total = float(pool_weight.sum()) cumulative = np.cumsum(pool_weight) @@ -281,6 +318,7 @@ def split_high_value_support_households( weights = np.asarray(frame.weights_for("household").values, dtype=float) old_total = frame.weights_for("household").total household_ids = household["household_id"].to_numpy() + tie_seed, tie_salt = high_value_support_tie_break() selected, region_rows = select_high_value_support_households( household_ids=household_ids, weights=weights, @@ -288,6 +326,8 @@ def split_high_value_support_households( tenure_type=household[_TENURE_COLUMN].to_numpy(), value=household[_VALUE_COLUMN].to_numpy(), stratum_lower=stratum_lower, + seed=tie_seed, + salt=tie_salt, ) copy_counts = np.ones(len(household), dtype="int64") copy_counts[selected] = int(copies) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py index 5476eeca2b..55910324e9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py @@ -61,7 +61,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -71,7 +71,10 @@ from microcosm.build.gates import FitWeightRecord from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.stochastic_assignment import ( + keyed_tie_order, + stable_identity_uniforms, +) from microcosm.build.uk_runtime.hmrc_property_rental import ( HMRC_PROPERTY_RENTAL_RECEIPTS_BAND_LOWER_BOUNDS, HMRC_PROPERTY_RENTAL_RECORD_SETS, @@ -259,14 +262,32 @@ def from_parameters(cls, parameters: Mapping[str, Any]) -> FinanceCostModelSpec: @dataclass(frozen=True) class ReceiptsAllocationSpec: - """The ``allocate_property_rental_income`` operation.""" + """The ``allocate_property_rental_income`` operation. + + ``tie_salt`` keys the walk's tie-break: landlords tied on profit take the + order of a uniform keyed on their person_id under ``walk_seed`` and this + salt, then of their ids (microcosm#931). + """ band_lower_bounds: tuple[int, ...] walk_seed: int + tie_salt: str maximum_top_band_multiplier: float @classmethod - def from_parameters(cls, parameters: Mapping[str, Any]) -> ReceiptsAllocationSpec: + def from_parameters( + cls, + parameters: Mapping[str, Any], + *, + salts_in_use: Collection[str] = (), + ) -> ReceiptsAllocationSpec: + """The declared operation, refused where it drifts from the stage's rules. + + ``salts_in_use`` names the salts the stage's other draws key on the + same person ids (the stage passes its finance-cost draws'): a tie + salt equal to one of them would order tied landlords by that draw. + """ + bounds = tuple(int(v) for v in parameters["band_lower_bounds"]) if bounds != HMRC_PROPERTY_RENTAL_RECEIPTS_BAND_LOWER_BOUNDS: raise PropertyComponentsError( @@ -286,9 +307,20 @@ def from_parameters(cls, parameters: Mapping[str, Any]) -> ReceiptsAllocationSpe raise PropertyComponentsError( "maximum_top_band_multiplier must be finite and above 1." ) + tie_salt = parameters.get("tie_salt") + if not isinstance(tie_salt, str) or not tie_salt: + raise PropertyComponentsError( + "allocate_property_rental_income must declare a non-empty tie_salt." + ) + if tie_salt in set(salts_in_use): + raise PropertyComponentsError( + "the receipts walk's tie_salt must differ from the salts of the " + "stage's other draws on the same person ids." + ) return cls( band_lower_bounds=bounds, walk_seed=int(parameters["walk_seed"]), + tie_salt=tie_salt, maximum_top_band_multiplier=multiplier, ) @@ -671,7 +703,9 @@ def allocate_property_rental_income( ) -> ReceiptsAllocation: """Receipts for every person with ``profit`` above zero (zero elsewhere). - Ranked by profit (person id breaks ties), walked top band first into + Ranked by profit, landlords tied on profit in the order of a uniform + keyed on their person id under the walk seed and the tie salt, then of + their ids (``keyed_tie_order``), walked top band first into Table 13's bands, each filled to its share of the individuals-basis landlords times the landlords' weight here, linear within a bounded band from its upper bound down, never below profit; the open top band takes @@ -691,7 +725,11 @@ def allocate_property_rental_income( top_down = list(reversed(bands)) if not len(landlord): raise PropertyComponentsError("no landlord with profit above zero to allocate.") - order = landlord[np.lexsort((ids[landlord], -profit[landlord]))] + order = landlord[ + keyed_tie_order( + -profit[landlord], ids[landlord], seed=spec.walk_seed, salt=spec.tie_salt + ) + ] ranked_weights = weights[order] landlord_weight = float(ranked_weights.sum()) published_landlords = float(sum(band.landlords for band in bands)) @@ -822,7 +860,11 @@ def mean_at(k: float) -> float: on = profit > 0.0 receipt = { "population": "persons with profit from property other than sub-letting above zero", - "order": "profit descending, person_id ascending", + "order": ( + "profit descending, landlords tied on profit in ascending order of a " + "uniform keyed on person_id under walk_seed and tie_salt, then " + "person_id ascending" + ), "walk": {"seed": spec.walk_seed, "offset": offset}, "band_basis": ( "PRIS 2024-25 Table 13 (all tax entities) x Table 1 individuals' " @@ -928,7 +970,8 @@ def __call__(self, frame: Frame) -> Frame: _operation(self.stage, IMPUTE_FRS_PROPERTY_FINANCE_COSTS_KIND) ) receipts_spec = ReceiptsAllocationSpec.from_parameters( - _operation(self.stage, ALLOCATE_PROPERTY_RENTAL_INCOME_KIND) + _operation(self.stage, ALLOCATE_PROPERTY_RENTAL_INCOME_KIND), + salts_in_use=(finance_spec.incidence_salt, finance_spec.amount_salt), ) verify = _operation(self.stage, VERIFY_VENDORED_FACT_RESOURCE_KIND) if ( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index 38db1087a7..d12f82f81e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -844,6 +844,13 @@ def _declared_seeds(stages) -> dict[str, dict[str, int]]: for stage in stages: stage_seeds: dict[str, int] = {} for operation in stage.operations: + if operation.kind == "allocate_property_rental_income": + # The receipts walk keys its tie-break on person ids under its + # walk seed and the tie salt (microcosm#931). + walk_seed = operation.parameters.get("walk_seed") + if isinstance(walk_seed, int): + stage_seeds[str(operation.parameters["tie_salt"])] = walk_seed + continue output = operation.parameters.get("output") seed = operation.parameters.get("seed") if seed is None: @@ -898,6 +905,13 @@ def _declared_seeds(stages) -> dict[str, dict[str, int]]: stage_seeds[str(operation.parameters["salt"])] = seed elif operation.kind == "top_up_to_stock": stage_seeds[str(operation.parameters["salt"])] = seed + elif operation.kind in ( + "split_top_wealth_support_households", + "split_top_value_owner_households", + ): + # The support splits' tie-breaks are keyed on household + # ids under their salts (microcosm#931). + stage_seeds[str(operation.parameters["salt"])] = seed elif operation.kind in ( "split_household_isa_balances", "allocate_isa_subscribers", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py index 1603c9e9e4..55c84e7b26 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py @@ -10,6 +10,8 @@ assign_binary_with_anchored_residual, clipped_normal_from_uniforms, integer_record_keys, + keyed_tie_order, + keyed_tie_uniforms, sample_categorical_from_counts, stable_identity_uniform, stable_identity_uniforms, @@ -337,3 +339,88 @@ def test_the_latent_attribute_band_is_the_shared_band_at_four_sigma() -> None: assert binomial_share_tolerance(target=target, rows=rows, sigma=4.0) == ( before ) + + +# --- keyed tie-breaks (microcosm#931) ---------------------------------------- + +_TIE = {"seed": 7, "salt": "test:tie"} + + +def _ties(n: int, rng: np.random.Generator) -> tuple[np.ndarray, np.ndarray]: + """Synthetic distinct integer ids and values drawn from a few atoms.""" + + ids = rng.choice(10**12, n, replace=False).astype(np.int64) + values = rng.integers(0, 6, n).astype(float) + return ids, values + + +def test_keyed_tie_order_is_the_id_order_where_nothing_ties() -> None: + rng = np.random.default_rng(3) + ids = rng.choice(10**12, 500, replace=False).astype(np.int64) + values = rng.permutation(500).astype(float) * 1.5 + + order = keyed_tie_order(values, ids, **_TIE) + + np.testing.assert_array_equal(order, np.lexsort((ids, values))) + np.testing.assert_array_equal( + keyed_tie_order(-values, ids, **_TIE), np.lexsort((ids, -values)) + ) + + +def test_keyed_tie_order_sorts_ties_on_their_uniforms_then_ids() -> None: + # The full form the helper's contract states: np.lexsort((ids, u, + # primary)) with u every record's uniform, though only tied records are + # drawn. NaNs tie with each other and -0.0 with 0.0. + rng = np.random.default_rng(4) + for trial in range(60): + ids, values = _ties(int(rng.integers(1, 80)), rng) + if trial % 3 == 0: + values[rng.random(values.size) < 0.2] = np.nan + if trial % 4 == 0: + values[values == 0.0] = -0.0 + uniforms = stable_identity_uniforms(ids, **_TIE) + + order = keyed_tie_order(values, ids, **_TIE) + + np.testing.assert_array_equal(order, np.lexsort((ids, uniforms, values))) + np.testing.assert_array_equal(np.sort(order), np.arange(values.size)) + np.testing.assert_array_equal( + keyed_tie_uniforms(ids, **_TIE), stable_identity_uniforms(ids, **_TIE) + ) + + +def test_keyed_tie_order_breaks_a_tie_otherwise_than_the_ids_would() -> None: + # Three records tie; their uniforms, not their ids, set their order. + ids = np.asarray([11, 12, 13, 14, 15], dtype=np.int64) + values = np.asarray([2.0, 1.0, 1.0, 1.0, 0.0]) + tied = ids[1:4] + keyed = tied[np.argsort(stable_identity_uniforms(tied, **_TIE))] + assert keyed.tolist() != sorted(keyed.tolist()) + + order = keyed_tie_order(values, ids, **_TIE) + + assert ids[order].tolist() == [15, *keyed.tolist(), 11] + + +def test_keyed_tie_order_does_not_depend_on_the_row_order() -> None: + rng = np.random.default_rng(5) + ids, values = _ties(300, rng) + expected = ids[keyed_tie_order(values, ids, **_TIE)] + for _ in range(5): + rows = rng.permutation(ids.size) + order = keyed_tie_order(values[rows], ids[rows], **_TIE) + np.testing.assert_array_equal(ids[rows][order], expected) + + +def test_keyed_tie_order_draws_on_its_salt_and_refuses_unusable_ids() -> None: + rng = np.random.default_rng(6) + ids, values = _ties(200, rng) + salted = keyed_tie_order(values, ids, seed=7, salt="test:other") + assert not np.array_equal(keyed_tie_order(values, ids, **_TIE), salted) + with pytest.raises(TypeError, match="integer ids"): + keyed_tie_order([1.0, 1.0], [1.0, 2.0], **_TIE) + with pytest.raises(ValueError, match="each record once"): + keyed_tie_order([1.0, 1.0], np.asarray([3, 3]), **_TIE) + with pytest.raises(ValueError, match="aligned"): + keyed_tie_order([1.0, 1.0, 2.0], np.asarray([3, 4]), **_TIE) + assert keyed_tie_order([], np.asarray([], dtype=np.int64), **_TIE).size == 0 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py index 794b43d455..ffdcfb9159 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py @@ -10,7 +10,10 @@ import pytest from microcosm.build.country_spec import load_country_spec -from microcosm.build.stochastic_assignment import stable_identity_uniform +from microcosm.build.stochastic_assignment import ( + stable_identity_uniform, + stable_identity_uniforms, +) from microcosm.build.uk_runtime import cgt_asset_type from microcosm.build.uk_runtime.cgt_asset_type import ( CGT_ASSET_TYPE_COLUMN, @@ -1118,6 +1121,107 @@ def test_refuses_a_frame_already_classified(self) -> None: assign_uk_cgt_asset_types(frame, facts, PARAMETERS, BADR_PARAMETERS) +class TestKeyedTies: + """The BADR walk and the main asset type walk order persons tied on gain + by a uniform keyed on their own person_id, then by id, each under its own + salt and the BADR seed (microcosm#931). Gains are plan amounts, so ties + are not expected on the spine; these frames force them.""" + + @staticmethod + def _tied_frame() -> tuple[Frame, HMRCCGTAssetTypeFacts]: + gains = _synthetic_gains() + tied = np.flatnonzero((gains > 12_000.0) & (gains < 20_000.0))[:30] + gains[tied] = 15_000.0 + weights = np.full(gains.size, 60.0) + return _frame(gains, weights=weights), _synthetic_facts(gains, weights) + + @staticmethod + def _recorded(monkeypatch, frame, facts, **kwargs): + calls: list[tuple[str, int, np.ndarray, np.ndarray, np.ndarray]] = [] + order = cgt_asset_type.keyed_tie_order + + def recording(primary, ids, *, seed, salt): + result = order(primary, ids, seed=seed, salt=salt) + calls.append((salt, seed, np.asarray(primary), np.asarray(ids), result)) + return result + + with monkeypatch.context() as patch: + patch.setattr(cgt_asset_type, "keyed_tie_order", recording) + result, _ = _assign(frame, facts, PARAMETERS, BADR_PARAMETERS, **kwargs) + return result, calls + + def test_both_walks_break_gain_ties_on_their_own_keyed_uniforms( + self, monkeypatch + ) -> None: + frame, facts = self._tied_frame() + _, calls = self._recorded(monkeypatch, frame, facts) + + salts = {salt for salt, *_ in calls} + assert salts == { + "assign_badr_qualifying_gains:tie:2024", + "assign_main_asset_type:tie:2024", + } + reordered = set() + for salt, seed, primary, ids, result in calls: + assert seed == 555 + uniforms = stable_identity_uniforms(ids, seed=seed, salt=salt) + np.testing.assert_array_equal(result, np.lexsort((ids, uniforms, primary))) + if not np.array_equal(result, np.lexsort((ids, primary))): + reordered.add(salt) + # Both walks met the forced tie and ordered it otherwise than by id. + assert reordered == salts + _, reseeded = self._recorded(monkeypatch, frame, facts, badr_seed=999) + assert {seed for _, seed, *_ in reseeded} == {999} + declared = cgt_asset_type_operation_parameters() + assert ( + "assign_badr_qualifying_gains:tie:" + in (declared["assign_badr_qualifying_gains"]["realization"]) + ) + main = declared["assign_main_asset_type"]["realization"] + assert "assign_main_asset_type:tie:" in main + assert "no seed" not in main + + def test_distinct_gains_assign_as_the_id_order_did(self, monkeypatch) -> None: + # The tie-break only reorders exact ties: on distinct gains both walks + # run in the id order's sequence and the stage's output and evidence + # are the same bit for bit. + gains = _synthetic_gains() + weights = np.full(gains.size, 60.0) + facts = _synthetic_facts(gains, weights) + frame = _frame(gains, weights=weights) + keyed, keyed_summary = _assign(frame, facts, PARAMETERS, BADR_PARAMETERS) + monkeypatch.setattr( + cgt_asset_type, + "keyed_tie_order", + lambda primary, ids, **_: np.lexsort((np.asarray(ids), primary)), + ) + by_id, by_id_summary = _assign(frame, facts, PARAMETERS, BADR_PARAMETERS) + + pd.testing.assert_frame_equal(keyed.table("person"), by_id.table("person")) + assert keyed_summary.evidence() == by_id_summary.evidence() + + def test_tied_gains_do_not_depend_on_the_row_order(self) -> None: + frame, facts = self._tied_frame() + person = frame.table("person") + order = np.random.default_rng(3).permutation(len(person)) + shuffled = uk_national_frame( + person=person.iloc[order].reset_index(drop=True), + benunit=frame.table("benunit"), + household=frame.table("household"), + time_period="2024", + household_weights=frame.weights_for("household").values, + ) + + first, _ = _assign(frame, facts, PARAMETERS, BADR_PARAMETERS) + second, _ = _assign(shuffled, facts, PARAMETERS, BADR_PARAMETERS) + + columns = [CGT_ASSET_TYPE_COLUMN, CGT_BADR_GAINS_COLUMN] + pd.testing.assert_frame_equal( + first.table("person").set_index("person_id").sort_index()[columns], + second.table("person").set_index("person_id").sort_index()[columns], + ) + + class TestStageContract: def test_manifest_operations_restate_the_reviewed_parameters(self) -> None: spec = load_country_spec("uk") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py index ba370d8c02..20bc317ae9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py @@ -1,7 +1,14 @@ """Tests split from packages/microcosm-build/tests/test_uk_cgt_imputation.py.""" # ruff: noqa: F403, F405 -from microcosm.build.stochastic_assignment import stable_identity_uniform +from microcosm.build.stochastic_assignment import ( + binomial_share_tolerance, + keyed_tie_order, + keyed_tie_uniforms, + stable_identity_uniform, + stable_identity_uniforms, +) +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_imputation import UK_CGT_IMPUTATION_SEED from test_support.microcosm_build.uk_cgt_imputation import * @@ -1744,3 +1751,226 @@ def test_the_redraw_does_not_depend_on_the_row_order(self) -> None: for label, (uniforms, members) in draws.items(): assert shuffled_draws[label][0] == uniforms, label np.testing.assert_array_equal(shuffled_draws[label][1], members) + + +#: The rank ties' salt on the helper frames, built for 2023. +_RANK_TIE = { + "seed": UK_CGT_IMPUTATION_SEED, + "salt": "hmrc_cgt_gains_spine:rank_tie:2023", +} + + +def _redrawn(frame: Frame) -> pd.Series: + result, _ = impute_uk_capital_gains_with_report( + frame, + _distribution(cell_people=300.0), + PARAMETERS, + conditioning=load_hmrc_cgt_conditioning_facts(), + ) + person = result.table("person") + return pd.Series( + person["capital_gains"].to_numpy(), index=person["person_id"].to_numpy() + ).sort_index() + + +class TestRankTies: + """Gainers tied on a rank take the order of one identity-keyed uniform + each, then of their person ids, not of their ids alone, which handed + every tie to the lowest id range: the first copy or clone of a household + (microcosm#931).""" + + def test_ranked_orders_ties_by_the_keyed_uniform_then_person_id(self) -> None: + person_id = np.asarray([40, 41, 42, 43, 44, 45, 46], dtype=np.int64) + key = np.asarray([0.9, 0.5, 0.5, 0.5, 0.5, 0.2, 0.7]) + tie = keyed_tie_uniforms(person_id, **_RANK_TIE) + indices = np.arange(len(key)) + + ranked = cgt_imputation._ranked( + indices, person_id=person_id, existing=key, tie=tie + ) + + # The shared helper's order on the negated key. + np.testing.assert_array_equal( + ranked, keyed_tie_order(-key, person_id, **_RANK_TIE) + ) + tied = person_id[ranked[2:6]] + assert tied.tolist() == person_id[1:5][np.argsort(tie[1:5])].tolist() + assert tied.tolist() != [41, 42, 43, 44] + # Without the uniforms, person_id alone breaks ties, as before. + np.testing.assert_array_equal( + cgt_imputation._ranked(indices, person_id=person_id, existing=key), + np.lexsort((person_id, -key)), + ) + # On distinct keys the uniforms move nothing. + distinct = np.asarray([0.9, 0.6, 0.5, 0.4, 0.3, 0.2, 0.7]) + np.testing.assert_array_equal( + cgt_imputation._ranked( + indices, person_id=person_id, existing=distinct, tie=tie + ), + np.lexsort((person_id, -distinct)), + ) + + def test_a_plan_and_the_remainder_give_ties_the_keyed_order_reversed( + self, + ) -> None: + # Both walk the rank ascending, so the member of a tie with the + # higher uniform takes the lower stratum, where the id order gave it + # to the higher person id. + person_id = np.arange(10, 16, dtype=np.int64) + existing = np.asarray([1.0, 2.0, 2.0, 2.0, 3.0, 4.0]) + weights = np.full(6, 100.0) + tie = keyed_tie_uniforms(person_id, **_RANK_TIE) + tied = np.asarray([1, 2, 3]) + keyed_low_first = tied[np.argsort(-tie[tied])] + assert keyed_low_first.tolist() != tied[::-1].tolist() + plan = _CellPlan( + 1_000_000, 2_000_000.0, 1_000_000.0, 9_000.0, 1_420_000.0, False, False + ) + + def strata(gains: np.ndarray) -> list[int]: + return tied[np.argsort(gains[tied])].tolist() + + for tie_argument, expected in ((tie, keyed_low_first), (None, tied[::-1])): + gains = np.zeros(6) + _draw_plan_amounts( + plan, + np.arange(6), + person_id=person_id, + existing=existing, + weights=weights, + rng=cgt_imputation._KeyedOffset(0.5), + new_gains=gains, + tie=tie_argument, + ) + assert strata(gains) == expected.tolist() + + surface = advani_summers_rows(load_advani_summers_distribution()) + band_index = advani_summers_band_index(surface, np.full(6, 30_000.0)) + for tie_argument, expected in ((tie, keyed_low_first), (None, tied[::-1])): + gains = existing.copy() + _map_remainder_amounts( + remainder=np.arange(6), + existing=existing, + person_id=person_id, + person_weight=weights, + band_index=band_index, + rows=surface, + annual_exempt_amount=3_000.0, + new_gains=gains, + tie=tie_argument, + ) + assert strata(gains) == expected.tolist() + + def test_clone_zero_ranks_first_among_tied_clones_at_its_share(self) -> None: + # Sixty source gainers, each as three geographic clones tied on the + # rank key: by person id, clone 0 ranked first in every family; keyed + # on each clone's own id, in about a third of them, inside the + # four-sigma binomial band (design verifier P10). + families, clone_count = 60, 3 + sources = np.arange(1, families + 1, dtype=np.int64) * 1_000 + 1 + person_id = np.concatenate( + [ + sources, + *( + copy_id_table.mint(sources, "geographic_support", k) + for k in range(1, clone_count) + ), + ] + ) + clone = np.repeat(np.arange(clone_count), families) + key = np.tile(np.linspace(0.1, 0.9, families), clone_count) + indices = np.arange(len(key)) + + def firsts(tie) -> np.ndarray: + ranked = cgt_imputation._ranked( + indices, person_id=person_id, existing=key, tie=tie + ) + blocks = ranked.reshape(families, clone_count) + assert (np.ptp(key[blocks], axis=1) == 0.0).all() + return clone[blocks[:, 0]] + + share = float((firsts(keyed_tie_uniforms(person_id, **_RANK_TIE)) == 0).mean()) + band = binomial_share_tolerance( + target=1 / clone_count, rows=families, sigma=4.0 + ) + assert abs(share - 1 / clone_count) <= band + assert float((firsts(None) == 0).mean()) == 1.0 + + def test_every_walk_plan_and_the_remainder_rank_on_one_keyed_draw( + self, monkeypatch + ) -> None: + # One uniform per person, keyed on person_id under the stage's seed + # and salted with the build period, reaches every _ranked call. + frame = _cells_frame((*_KEYED_CELLS, _KEYED_EARLY_CELL)) + calls: list[np.ndarray | None] = [] + ranked = cgt_imputation._ranked + + def recording(indices, **kwargs): + calls.append(kwargs["tie"]) + return ranked(indices, **kwargs) + + monkeypatch.setattr(cgt_imputation, "_ranked", recording) + _redrawn(frame) + + person_id = frame.table("person")["person_id"].to_numpy() + expected = stable_identity_uniforms(person_id, **_RANK_TIE) + # Six cell walks, the pooled walk, the plans and the remainder's bands. + assert len(calls) > 7 + for tie in calls: + np.testing.assert_array_equal(tie, expected) + + def test_tied_rank_keys_redraw_whatever_the_row_order(self) -> None: + # Wealth in six-row families, as the high-value copies carry it, ties + # the blended rank key; the redraw is still the same, gainer by + # gainer, under a row permutation. + rng = np.random.default_rng(44) + rows = 1_200 + wealth = np.repeat(rng.lognormal(11.0, 1.5, rows // 6), 6) + frame = _frame( + rows, + gains=np.where(rng.random(rows) < 0.6, rng.lognormal(10, 1.4, rows), 0.0), + incomes=rng.lognormal(10.6, 0.7, rows), + wealth=wealth, + weights=rng.uniform(50.0, 400.0, rows), + ) + person = frame.table("person") + gainer = person["capital_gains"].to_numpy() > 0.0 + key, _ = cgt_imputation._allocation_rank_key( + person, frame.table("household"), person["capital_gains"].to_numpy() + ) + assert pd.Series(key[gainer]).duplicated().any() + order = np.random.default_rng(5).permutation(rows) + shuffled = uk_national_frame( + person=person.iloc[order].reset_index(drop=True), + benunit=frame.table("benunit"), + household=frame.table("household"), + time_period="2023", + household_weights=frame.weights_for("household").values, + ) + + def redrawn(frame: Frame) -> pd.Series: + result, _ = impute_uk_capital_gains_with_report( + frame, + _real_distribution(), + PARAMETERS, + conditioning=load_hmrc_cgt_conditioning_facts(), + ) + drawn = result.table("person") + return pd.Series( + drawn["capital_gains"].to_numpy(), index=drawn["person_id"].to_numpy() + ).sort_index() + + pd.testing.assert_series_equal( + redrawn(shuffled), redrawn(frame), check_exact=True + ) + + def test_untied_rank_keys_redraw_as_the_id_order_did(self, monkeypatch) -> None: + # Equal wealth leaves the prior ranking in charge, and distinct priors + # tie no key, so the redraw is the id order's bit for bit; the + # uniforms reorder exact ties alone. + frame = _cells_frame((*_KEYED_CELLS, _KEYED_EARLY_CELL)) + keyed = _redrawn(frame) + monkeypatch.setattr( + cgt_imputation, "keyed_tie_uniforms", lambda *args, **kwargs: None + ) + pd.testing.assert_series_equal(_redrawn(frame), keyed, check_exact=True) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_source_manifest.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_source_manifest.py index f2f1778ad6..6a167045f0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_source_manifest.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_source_manifest.py @@ -67,6 +67,18 @@ def test_manifest_operations_match_the_stage_implementation() -> None: draws = operations["within_band_draws"] assert draws["seed_base"] == UK_CGT_IMPUTATION_SEED assert draws["deterministic"] is True + # Ties on a rank take one uniform per person under the seed, salted with + # the period, the remainder's in reverse (microcosm#931). + ordering = operations["rank_preserving_allocation"]["ordering"] + assert ordering.startswith( + "a rank key descending, then the rank-tie uniform ascending, then " + "person_id ascending:" + ) + assert "salted hmrc_cgt_gains_spine:rank_tie:" in ordering + assert operations["sub_aea_remainder"]["ordering"].startswith( + "existing gains ascending, then the rank-tie uniform descending, then " + "person_id descending" + ) verify = operations["verify_vendored_fact_resource"] assert verify["require_before_source_read"] is True assert verify["fail_on_mismatch"] is True diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 538bf86425..39e133b367 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -13,7 +13,11 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceOperationSpec -from microcosm.build.uk_runtime import cgt_support +from microcosm.build.stochastic_assignment import ( + binomial_share_tolerance, + stable_identity_uniforms, +) +from microcosm.build.uk_runtime import cgt_support, copy_id_table from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_INVESTABLE_WEALTH_COLUMNS, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, @@ -37,6 +41,8 @@ CGT_SUPPORT_PUBLISHED_TOP_BAND_TAXPAYERS, CGT_SUPPORT_SPLIT_OPERATION_KIND, CGT_SUPPORT_SPLIT_STAGE_NAME, + CGT_SUPPORT_TIE_SALT, + CGT_SUPPORT_TIE_SEED, HOUSEHOLD_IS_CGT_SUPPORT_COPY, UKCGTSupportSplitStageTransform, _assert_cgt_support_split_stage_parameters, @@ -46,6 +52,7 @@ cgt_support_income_band, cgt_support_mass_by_income_band, cgt_support_split_operation_parameters, + cgt_support_tie_break, geographic_clone_count, select_cgt_support_households, split_cgt_support_households, @@ -83,12 +90,13 @@ # 20,000): ids 1-4 and the zero-weight id 7; band 37,700 (income 60,000): # ids 5-6. The synthetic joint publishes 60 top-band taxpayers in band 0 # (support mass 2 x 2 x 60 = 240, one suppressed cell) and 10 in band 37,700 -# (support mass 40). Band 0 walks by wealth descending then id ascending: id -# 2 (500, 130) then id 3 (500, 121; the tie breaks on id) reach 251, so id 3 -# is the crossing household and ids 4 (200) and 1 (100) stay. Band 37,700 -# walks id 6 (20, 61) first, which crosses at once; id 5 (10, 90) stays and -# is the heaviest unselected incumbent, so the exact-total correction lands -# on it. Split: id 2 into 3 copies, id 3 into 3, id 6 into 2. +# (support mass 40). Band 0 walks by wealth descending, ties by the walk's +# keyed uniforms: ids 2 (500, 130) and 3 (500, 121) tie, and together reach +# 251, so both are taken whichever crosses (id 3's uniform is the lower, so +# id 2 crosses), and ids 4 (200) and 1 (100) stay. Band 37,700 walks id 6 +# (20, 61) first, which crosses at once; id 5 (10, 90) stays and is the +# heaviest unselected incumbent, so the exact-total correction lands on it. +# Split: id 2 into 3 copies, id 3 into 3, id 6 into 2. SCENARIO_WEIGHTS = [70.0, 130.0, 121.0, 30.0, 90.0, 61.0, 0.0] SCENARIO_WEALTH = [100.0, 500.0, 500.0, 200.0, 10.0, 20.0, 1_000_000.0] @@ -114,6 +122,8 @@ #: The copy-id table's cgt_support_split radix (microcosm#931). COPY_RADIX = radix("cgt_support_split") +#: The walk's declared tie-break, which every direct call passes. +TIE = {"seed": CGT_SUPPORT_TIE_SEED, "salt": CGT_SUPPORT_TIE_SALT} def _scenario_frame(**overrides): @@ -322,20 +332,23 @@ def test_household_wealth_sums_the_investable_columns_and_refuses_gaps(): # --- selection ---------------------------------------------------------------- -def test_selection_walks_wealth_descending_then_id_ascending_whole_households(): +def test_selection_walks_wealth_descending_ties_keyed_whole_households(): ids = np.asarray([1, 2, 3, 4, 5, 6, 7]) weights = np.asarray(SCENARIO_WEIGHTS) bands = np.asarray([0, 0, 0, 0, 37_700, 37_700, 0]) wealth = np.asarray(SCENARIO_WEALTH) selected, rows = select_cgt_support_households( - ids, weights, bands, wealth, SCENARIO_SUPPORT + ids, weights, bands, wealth, SCENARIO_SUPPORT, **TIE ) - # Band 0 at 240: id 2 (500, 130) then id 3 (500, 121; the tie breaks on - # id) reach 251 and id 3 is the crossing household, taken whole; id 4 - # (200) and id 1 (100) stay. Band 37,700 at 40: id 6 (20, 61) crosses at - # once; id 5 (10, 90) stays. + # Band 0 at 240: ids 2 (500, 130) and 3 (500, 121) tie on wealth and + # together reach 251, so both are taken in either order (id 3's keyed + # uniform is the lower, so id 2 crosses); id 4 (200) and id 1 (100) + # stay. Band 37,700 at 40: id 6 (20, 61) crosses at once; id 5 (10, 90) + # stays. + uniforms = stable_identity_uniforms([2, 3], **TIE) + assert uniforms[1] < uniforms[0] assert ids[selected].tolist() == [2, 3, 6] band_0, band_37700 = rows assert band_0 == { @@ -370,13 +383,13 @@ def test_selection_takes_the_crossing_household_whole_and_stops_there(): for support, expected in ((0.0, []), (1.0, [1]), (10.0, [1]), (10.5, [1, 2])): selected, (row,) = select_cgt_support_households( - ids, weights, bands, wealth, {0: support} + ids, weights, bands, wealth, {0: support}, **TIE ) assert ids[selected].tolist() == expected assert row["households_selected"] == len(expected) assert row["pool_exhausted"] is False selected, (row,) = select_cgt_support_households( - ids, weights, bands, wealth, {0: 40.0} + ids, weights, bands, wealth, {0: 40.0}, **TIE ) assert ids[selected].tolist() == [1, 2, 3, 4] assert row["pool_exhausted"] is False @@ -389,7 +402,7 @@ def test_selection_records_pool_exhaustion_and_empty_bands_without_refusing(): wealth = np.asarray([1.0, 2.0, 3.0]) selected, rows = select_cgt_support_households( - ids, weights, bands, wealth, {0: 1_000.0, 50_000: 40.0, 200_000: 0.0} + ids, weights, bands, wealth, {0: 1_000.0, 50_000: 40.0, 200_000: 0.0}, **TIE ) assert selected.tolist() == [True, True, True] @@ -419,7 +432,7 @@ def test_selection_excludes_zero_weight_rows_and_is_permutation_stable(): bands = np.asarray([0, 0, 0, 0, 37_700, 37_700, 0]) wealth = np.asarray(SCENARIO_WEALTH) selected, rows = select_cgt_support_households( - ids, weights, bands, wealth, SCENARIO_SUPPORT + ids, weights, bands, wealth, SCENARIO_SUPPORT, **TIE ) # Id 7 is the wealthiest household of band 0 and weighs nothing. assert not selected[6] @@ -432,6 +445,7 @@ def test_selection_excludes_zero_weight_rows_and_is_permutation_stable(): bands[permutation], wealth[permutation], SCENARIO_SUPPORT, + **TIE, ) assert permuted.tolist() == selected[permutation].tolist() @@ -439,18 +453,182 @@ def test_selection_excludes_zero_weight_rows_and_is_permutation_stable(): def test_selection_refuses_inconsistent_inputs(): + def select(*args): + return select_cgt_support_households(*args, **TIE) + with pytest.raises(ValueError, match="one entry per household"): - select_cgt_support_households([1, 2], [1.0], [0, 0], [0.0, 0.0], {0: 1.0}) + select([1, 2], [1.0], [0, 0], [0.0, 0.0], {0: 1.0}) with pytest.raises(ValueError, match="unique household ids"): - select_cgt_support_households([1, 1], [1.0, 1.0], [0, 0], [0.0, 0.0], {0: 1.0}) + select([1, 1], [1.0, 1.0], [0, 0], [0.0, 0.0], {0: 1.0}) with pytest.raises(ValueError, match="finite and non-negative"): - select_cgt_support_households([1, 2], [1.0, -1.0], [0, 0], [0.0, 0.0], {0: 1.0}) + select([1, 2], [1.0, -1.0], [0, 0], [0.0, 0.0], {0: 1.0}) with pytest.raises(ValueError, match="wealth must be finite"): - select_cgt_support_households( - [1, 2], [1.0, 1.0], [0, 0], [0.0, np.nan], {0: 1.0} - ) + select([1, 2], [1.0, 1.0], [0, 0], [0.0, np.nan], {0: 1.0}) with pytest.raises(ValueError, match="support mass"): - select_cgt_support_households([1, 2], [1.0, 1.0], [0, 0], [0.0, 0.0], {0: -1.0}) + select([1, 2], [1.0, 1.0], [0, 0], [0.0, 0.0], {0: -1.0}) + # The tie-break has no default: every caller passes the declared pair. + with pytest.raises(TypeError, match="seed"): + select_cgt_support_households([1, 2], [1.0, 1.0], [0, 0], [0.0, 0.0], {0: 1.0}) + + +def _id_walk(ids, weights, bands, wealth, support_by_band) -> np.ndarray: + """The selection as the walk made it before (microcosm#931): ties by id.""" + + selected = np.zeros(len(ids), dtype=bool) + for band, target in support_by_band.items(): + pool = np.flatnonzero((bands == band) & (weights > 0.0)) + order = pool[np.lexsort((ids[pool], -wealth[pool]))] + reached = np.flatnonzero(np.cumsum(weights[order]) >= target) + take = int(reached[0]) + 1 if target > 0.0 and reached.size else 0 + if target > 0.0 and not reached.size: + take = len(order) + selected[order[:take]] = True + return selected + + +def test_a_wealth_tie_straddling_the_support_mass_is_walked_in_keyed_order(): + # Four households tie on wealth across the band's support mass of 25: + # after the wealthiest (10), the walk takes the first two of the tie + # group (20 and 30), in the order of their keyed uniforms, which is not + # their id order, then stops; the poorest stays (microcosm#931). + tied = np.asarray([201, 202, 203, 204]) + ids = np.asarray([101, *tied, 301]) + weights = np.full(6, 10.0) + bands = np.zeros(6, dtype=int) + wealth = np.asarray([9.0, 5.0, 5.0, 5.0, 5.0, 1.0]) + keyed = tied[np.lexsort((tied, stable_identity_uniforms(tied, **TIE)))] + assert sorted(keyed[:2].tolist()) != [201, 202] + + selected, (row,) = select_cgt_support_households( + ids, weights, bands, wealth, {0: 25.0}, **TIE + ) + + assert sorted(ids[selected].tolist()) == sorted([101, *keyed[:2].tolist()]) + assert row["households_selected"] == 3 and row["selected_mass"] == 30.0 + assert ids[_id_walk(ids, weights, bands, wealth, {0: 25.0})].tolist() == [ + 101, + 201, + 202, + ] + for seed in range(5): + rows = np.random.default_rng(seed).permutation(6) + permuted, (permuted_row,) = select_cgt_support_households( + ids[rows], weights[rows], bands[rows], wealth[rows], {0: 25.0}, **TIE + ) + assert permuted.tolist() == selected[rows].tolist() + assert permuted_row == row + + +def test_distinct_wealth_selects_as_the_id_walk_did_bit_for_bit(): + # The tie-break only reorders exact ties, so on distinct wealth every + # band takes the households the walk took before (microcosm#931). + rng = np.random.default_rng(20261109) + n = 600 + ids = rng.choice(10**8, n, replace=False).astype(np.int64) + weights = rng.uniform(1.0, 400.0, n) + weights[rng.random(n) < 0.05] = 0.0 + bands = rng.choice(CGT_SUPPORT_INCOME_BAND_LOWER_BOUNDS, n) + wealth = rng.permutation(n) * 1_000.0 + rng.random(n) + support = { + int(band): float(mass) + for band, mass in zip( + CGT_SUPPORT_INCOME_BAND_LOWER_BOUNDS, + rng.uniform(100.0, 9_000.0, len(CGT_SUPPORT_INCOME_BAND_LOWER_BOUNDS)), + strict=True, + ) + } + + selected, rows = select_cgt_support_households( + ids, weights, bands, wealth, support, **TIE + ) + + np.testing.assert_array_equal( + selected, _id_walk(ids, weights, bands, wealth, support) + ) + assert 0 < int(selected.sum()) < n + rows_order = rng.permutation(n) + permuted, permuted_rows = select_cgt_support_households( + ids[rows_order], + weights[rows_order], + bands[rows_order], + wealth[rows_order], + support, + **TIE, + ) + assert permuted.tolist() == selected[rows_order].tolist() + # The pool's mass is summed in row order, so it alone may move in its + # last bits. + for permuted_row, row in zip(permuted_rows, rows, strict=True): + assert permuted_row["pool_mass"] == pytest.approx(row["pool_mass"], rel=1e-12) + assert {**permuted_row, "pool_mass": None} == {**row, "pool_mass": None} + + +def test_clone_zero_is_taken_from_tied_clones_at_its_share(): + # Sixty source households, each as three geographic clones tied on wealth + # (ids minted in the copy-id table's geographic field), each alone in its + # band at a support mass one clone's weight covers: every band takes one + # clone. The walk by id took clone 0 every time; keyed on each clone's own + # id, clone 0 is taken at about a third of the bands, inside the four-sigma + # binomial band (design verifier P10; microcosm#931). + families, clone_count = 60, 3 + sources = np.arange(1, families + 1, dtype=np.int64) + ids = np.concatenate( + [ + sources, + *( + copy_id_table.mint(sources, "geographic_support", k) + for k in range(1, clone_count) + ), + ] + ) + clone = np.repeat(np.arange(clone_count), families) + bands = np.tile(sources, clone_count) + wealth = np.tile(1_000.0 * sources, clone_count) + weights = np.full(families * clone_count, 10.0) + support = {int(band): 5.0 for band in sources} + + selected, rows = select_cgt_support_households( + ids, weights, bands, wealth, support, **TIE + ) + + assert [row["households_selected"] for row in rows] == [1] * families + share = float((clone[selected] == 0).mean()) + band = binomial_share_tolerance(target=1 / clone_count, rows=families, sigma=4.0) + assert abs(share - 1 / clone_count) <= band + by_id = _id_walk(ids, weights, bands, wealth, support) + assert float((clone[by_id] == 0).mean()) == 1.0 + + +def test_the_split_and_its_rerun_walk_on_the_declared_tie_break(monkeypatch): + # The stage's walk and keeps_the_split's rerun pass the seed and salt the + # declaration names, never a default (microcosm#931). + calls: list[dict[str, object]] = [] + select = cgt_support.select_cgt_support_households + + def recording(*args, **kwargs): + calls.append(dict(kwargs)) + return select(*args, **kwargs) + + monkeypatch.setattr(cgt_support, "select_cgt_support_households", recording) + frame = support_frame( + weights=[145.5, 37.87, 7.05, 26.65, 32.73, 37.87], + wealth=[9e6, 8e6, 4e3, 3e3, 2e3, 1.0], + incomes=[BAND_INCOMES[0]] * 6, + ) + split_cgt_support_households( + frame, + distribution=support_distribution({0: {250_000: 65.77}}), + parameters=PARAMETERS, + clone_count=1, + ) + + ((_, payload),) = cgt_support_split_operation_parameters() + assert cgt_support_tie_break() == (payload["seed"], payload["salt"]) + assert cgt_support_tie_break() == (0, "cgt_support_split:wealth_tie") + assert len(calls) == 2 + assert all( + (call["seed"], call["salt"]) == cgt_support_tie_break() for call in calls + ) # --- copy counts -------------------------------------------------------------- @@ -1818,9 +1996,10 @@ def test_the_walk_can_stop_partway_through_a_households_clones(): # support mass of 1,500 (2 x 1.25 x 600 published). At one clone the walk # takes the first two, at 15 and 30 copies. At three it accumulates each # clone's own weight: the first household's three clones (900 together) - # and then the second's clone 0 (600) reach 1,500, so the second's clones - # 1 and 2 stay whole. Selection nests only approximately (microcosm#931); - # every clone the walk selects gets its source's count. + # and then the first of the second's clones, which tie on wealth, in + # keyed order (600) reach 1,500, so its other two clones stay whole. + # Selection nests only approximately (microcosm#931); every clone the + # walk selects gets its source's count. def counts_by_clone(clone_count: int): frame = clone_first_frame( support_frame( @@ -1842,14 +2021,16 @@ def counts_by_clone(clone_count: int): one, one_selected = counts_by_clone(1) three, three_selected = counts_by_clone(3) + second = 2 + np.arange(3) * copy_id_table.radix("geographic_support") + first = int(np.argmin(stable_identity_uniforms(second, **TIE))) + # The walk by id took clone 0; the keyed order takes another here. + assert first != 0 assert one == {(1, 0): 15, (2, 0): 30, (3, 0): 1} assert one_selected == 2 assert three == { **{(1, clone): 15 for clone in range(3)}, - (2, 0): 30, - (2, 1): 1, - (2, 2): 1, + **{(2, clone): 30 if clone == first else 1 for clone in range(3)}, **{(3, clone): 1 for clone in range(3)}, } assert three_selected == 4 @@ -2133,6 +2314,21 @@ def test_operation_dictionary_restates_the_reviewed_design(): UK_CGT_INVESTABLE_WEALTH_COLUMNS ) assert payload["maximum_copy_weight"] == 60.0 + # The walk's tie-break is keyed on each household's own id under its own + # seed and salt, its one draw (microcosm#931). + assert payload["selection"] == ( + "within each income band walk households in investable wealth " + "descending, households tied on it in ascending order of a uniform " + "keyed on their own household_id under seed and salt, then household_id " + "ascending, accumulating positive household weight until the cumulative " + "weight reaches the band's support mass; whole households; an exhausted " + "pool is recorded, not refused" + ) + assert (payload["seed"], payload["salt"]) == (0, "cgt_support_split:wealth_tie") + assert payload["values"] == ( + "every column copied unchanged; the selection's tie-break is the stage's " + "only draw" + ) assert payload["copy_rule"] == ( "copies = ceil(geographic clone count x household weight / " "maximum_copy_weight), the quotient snapped to the nearest integer " @@ -2173,6 +2369,8 @@ def test_operation_dictionary_restates_the_reviewed_design(): "wealth_measure", "investable_wealth_columns", "selection", + "seed", + "salt", "maximum_copy_weight", "copy_rule", "id_remapping", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index fbd8dfcf1c..b69bf48539 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -4633,7 +4633,13 @@ def test_e8_manifest_seeds_all_reach_the_build_sidecar_harvester() -> None: "uc_deduction_random_draw": 0, "uc_deduction_type_random_draw": 0, } - assert "cgt_support_split" not in declared + # The support splits' and the receipts walk's tie-breaks, under their + # salts (microcosm#931). + assert declared["cgt_support_split"] == {"cgt_support_split:wealth_tie": 0} + assert declared["high_value_home_support"] == { + "high_value_home_support:value_tie": 0 + } + assert declared["property_components"] == {"allocate_property_rental_income:tie": 0} assert declared["spi_income_band_donors"] == {"spi_income_band_donor_draw": 3} assert declared["hmrc_spi_income_spine"]["spi_income_band_donor_leaf_draw"] == 44 assert declared["hmrc_cgt_gains_spine"] == {"within_band_draws": 552} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py index 3150fe8980..90e9d25ac7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py @@ -7,6 +7,11 @@ import pytest from microcosm.build.country_spec import load_country_spec +from microcosm.build.stochastic_assignment import ( + binomial_share_tolerance, + stable_identity_uniforms, +) +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime import high_value_home_support as hv from microcosm.build.uk_runtime.household_lineage import ( HIGH_VALUE_SUPPORT_COPY_COLUMN, @@ -21,6 +26,9 @@ assert_copy_ids_follow_lineage, ) +#: The stratum's declared tie-break, which every direct call passes. +TIE = {"seed": hv.HIGH_VALUE_SUPPORT_TIE_SEED, "salt": hv.HIGH_VALUE_SUPPORT_TIE_SALT} + def _frame(n: int = 40): """``n`` single-adult owner households in one region, values ascending.""" @@ -71,6 +79,7 @@ def test_selection_takes_the_top_stratum_of_each_region() -> None: tenure_type=household["tenure_type"].to_numpy(), value=household["main_residence_value"].to_numpy(), stratum_lower=0.9, + **TIE, ) # 38 equal-weight owners: mid-ranks (i + 0.5) / 38 >= 0.9 for the top four. assert int(selected.sum()) == 4 @@ -88,6 +97,166 @@ def test_selection_takes_the_top_stratum_of_each_region() -> None: ) +def _owners(values, *, ids=None, region="LONDON"): + """Equal-weight owners in one region, the selection's inputs.""" + + n = len(values) + return { + "household_ids": np.arange(1, n + 1, dtype=np.int64) if ids is None else ids, + "weights": np.full(n, 10.0), + "region": np.full(n, region, dtype=object), + "tenure_type": np.full(n, "OWNED_OUTRIGHT", dtype=object), + "value": np.asarray(values, dtype=float), + } + + +def _id_ranked_stratum(inputs, stratum_lower: float) -> np.ndarray: + """The stratum as the rank took it before (microcosm#931): ties by id.""" + + ids = np.asarray(inputs["household_ids"]) + weight = np.asarray(inputs["weights"], dtype=float) + region = np.asarray(inputs["region"]).astype(str) + values = np.asarray(inputs["value"], dtype=float) + selected = np.zeros(len(ids), dtype=bool) + for code in sorted(set(region)): + pool = np.flatnonzero((region == code) & (values > 0) & (weight > 0)) + order = pool[np.lexsort((ids[pool], values[pool]))] + cumulative = np.cumsum(weight[order]) + mid_rank = (cumulative - weight[order] / 2.0) / float(weight[order].sum()) + selected[order[mid_rank >= stratum_lower]] = True + return selected + + +def test_a_value_tie_straddling_the_stratum_is_ranked_in_keyed_order() -> None: + # Twenty equal-weight owners, the top three tied on value: at 0.9 the two + # highest mid-ranks are selected, so the cut falls inside the tie group, + # and the two of the three ranked last in keyed order are selected, not + # the two highest ids (microcosm#931). + values = [*(100_000.0 * (i + 1) for i in range(17)), *([3_000_000.0] * 3)] + inputs = _owners(values) + tied = np.asarray([18, 19, 20]) + keyed = tied[np.lexsort((tied, stable_identity_uniforms(tied, **TIE)))] + assert sorted(keyed[1:].tolist()) != [19, 20] + + selected, (row,) = hv.select_high_value_support_households( + **inputs, stratum_lower=0.9, **TIE + ) + + ids = inputs["household_ids"] + assert sorted(ids[selected].tolist()) == sorted(keyed[1:].tolist()) + assert row["households_selected"] == 2 and row["selected_mass"] == 20.0 + assert ids[_id_ranked_stratum(inputs, 0.9)].tolist() == [19, 20] + for seed in range(5): + rows = np.random.default_rng(seed).permutation(len(values)) + permuted, (permuted_row,) = hv.select_high_value_support_households( + **{key: np.asarray(value)[rows] for key, value in inputs.items()}, + stratum_lower=0.9, + **TIE, + ) + assert permuted.tolist() == selected[rows].tolist() + assert permuted_row == row + + +def test_distinct_values_rank_the_stratum_as_the_id_rank_did_bit_for_bit() -> None: + # The tie-break only reorders exact ties, so on distinct values every + # region's stratum is the one the id rank took, down to the household + # whose mid-rank sits nearest the cut, in each of forty regions + # (microcosm#931). + rng = np.random.default_rng(20261109) + n = 2_000 + inputs = { + "household_ids": rng.choice(10**8, n, replace=False).astype(np.int64), + "weights": rng.uniform(1.0, 50.0, n), + "region": rng.choice([f"R{code:02d}" for code in range(40)], n).astype(object), + "tenure_type": np.full(n, "OWNED_OUTRIGHT", dtype=object), + "value": rng.permutation(n) * 1_000.0 + 50_000.0, + } + + selected, _ = hv.select_high_value_support_households(**inputs, **TIE) + + np.testing.assert_array_equal( + selected, _id_ranked_stratum(inputs, hv.HIGH_VALUE_SUPPORT_STRATUM_LOWER) + ) + assert 0 < int(selected.sum()) < n + + +def test_tied_clones_enter_the_stratum_at_their_share() -> None: + # Sixty regions, each of seventeen owners below one source household's + # three geographic clones, tied on value at the top: the stratum takes + # one clone a region. The rank by id took the last clone every time; + # keyed on each clone's own id, clone 0 enters at about a third of the + # regions, inside the four-sigma binomial band (design verifier P10; + # microcosm#931). + families, clone_count = 60, 3 + parts: list[dict[str, np.ndarray]] = [] + clone_index: list[np.ndarray] = [] + for family in range(families): + source = np.int64(10_000 + family) + clones = np.asarray( + [ + source, + *( + copy_id_table.mint(np.asarray([source]), "geographic_support", k)[0] + for k in range(1, clone_count) + ), + ], + dtype=np.int64, + ) + below = np.arange(1, 18, dtype=np.int64) + 100 * family + parts.append( + _owners( + [*(1_000.0 * (i + 1) for i in range(17)), *([90_000.0] * clone_count)], + ids=np.concatenate([below, clones]), + region=f"R{family:02d}", + ) + ) + clone_index.append(np.asarray([-1] * 17 + list(range(clone_count)))) + inputs = {key: np.concatenate([part[key] for part in parts]) for key in parts[0]} + clone = np.concatenate(clone_index) + + selected, rows = hv.select_high_value_support_households(**inputs, **TIE) + + assert [row["households_selected"] for row in rows] == [1] * families + share = float((clone[selected] == 0).mean()) + band = binomial_share_tolerance(target=1 / clone_count, rows=families, sigma=4.0) + assert abs(share - 1 / clone_count) <= band + by_id = _id_ranked_stratum(inputs, hv.HIGH_VALUE_SUPPORT_STRATUM_LOWER) + assert float((clone[by_id] == clone_count - 1).mean()) == 1.0 + + +def test_the_stage_ranks_on_the_declared_tie_break(monkeypatch) -> None: + calls: list[dict[str, object]] = [] + select = hv.select_high_value_support_households + + def recording(**kwargs): + calls.append(kwargs) + return select(**kwargs) + + monkeypatch.setattr(hv, "select_high_value_support_households", recording) + hv.split_high_value_support_households(_frame(), stratum_lower=0.9) + + ((_, payload),) = hv.high_value_support_operation_parameters() + assert hv.high_value_support_tie_break() == (payload["seed"], payload["salt"]) + assert hv.high_value_support_tie_break() == (0, "high_value_home_support:value_tie") + (call,) = calls + assert (call["seed"], call["salt"]) == hv.high_value_support_tie_break() + assert payload["stratum_rule"] == ( + "within each FRS region, owner households (tenure owned outright or with " + "a mortgage, main_residence_value > 0, positive weight) ordered by " + "main_residence_value ascending, households tied on it in ascending " + "order of a uniform keyed on their own household_id under seed and salt, " + "then household_id ascending; a household whose weighted mid-rank " + "(cumulative weight before it plus half its own, over the region's owner " + "weight) is at or above stratum_lower is selected" + ) + assert payload["values"] == ( + "every column copied unchanged; the stratum's tie-break is the stage's " + "only draw" + ) + with pytest.raises(TypeError, match="seed"): + select(**_owners([1.0, 2.0])) + + def test_split_copies_whole_households_and_conserves_mass() -> None: frame = _frame() result = hv.split_high_value_support_households(frame, stratum_lower=0.9, copies=3) @@ -476,6 +645,13 @@ def test_stage_manifest_restates_the_code_dictionary() -> None: ) transform = hv.UKHighValueHomeSupportStageTransform(stage=stage) assert transform.output_columns() == tuple(stage.outputs) + # The manifest declares the stratum's tie-break the stage walks on + # (microcosm#931). + (operation,) = stage.operations + assert ( + operation.parameters["seed"], + operation.parameters["salt"], + ) == hv.high_value_support_tie_break() names = [s.stage for s in spec.sources.stages] assert names.index("regional_property_uprating") + 1 == names.index( hv.HIGH_VALUE_SUPPORT_STAGE_NAME diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index b0775c29a7..509e905ae8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -1352,6 +1352,49 @@ class TestSupportSplitRecomputeAcrossGeographicClones: goes unreported. """ + def test_the_recompute_walks_a_tie_at_the_crossing_as_the_stage_did( + self, monkeypatch + ) -> None: + # At three clones the second household's clones tie on wealth across + # band 0's support mass of 1,500, and the stage takes the one its keyed + # uniform puts first, not clone 0 (microcosm#931). The recompute walks + # under the declared tie-break too, and agrees; walked by id, as the + # rule before did, it takes clone 0 and reports the family the stage + # made as missing and clone 0 as one it did not make. + from microcosm.build.uk_runtime import cgt_support + + scenario = { + "weights": [900.0, 1_800.0, 600.0], + "wealth": [3e3, 2e3, 1e3], + "incomes": [BAND_INCOMES[0]] * 3, + "top_cells": {0: {250_000: 600.0}}, + } + split, frame = _clone_first_split_then_cloned(3, scenario) + assert split.households_selected == 4 + household = frame.table("household") + families = household.loc[ + household[CGT_SUPPORT_COPIES_COLUMN] == 30, "household_clone_index" + ] + assert set(families) == {1} + calls: list[tuple[int, str]] = [] + select = cgt_support.select_cgt_support_households + + def recording(*args, **kwargs): + calls.append((kwargs["seed"], kwargs["salt"])) + return select(*args, **kwargs) + + monkeypatch.setattr(cgt_support, "select_cgt_support_households", recording) + _, problems = _exhausting_receipt(frame, scenario) + assert problems == {} + assert calls and set(calls) == {cgt_support.cgt_support_tie_break()} + monkeypatch.setattr( + cgt_support, + "keyed_tie_order", + lambda primary, ids, **_: np.lexsort((np.asarray(ids), primary)), + ) + _, by_id = _exhausting_receipt(frame, scenario) + assert by_id["support_split_selection_stored"] == {"missing": 1, "extra": 1} + def test_at_one_clone_the_receipt_is_the_rule_before_the_clone_count( self, monkeypatch ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py index 9903afbe2d..0e63d143d6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py @@ -2,11 +2,20 @@ from __future__ import annotations +from dataclasses import replace + import numpy as np import pandas as pd import pytest from microcosm.build import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.stochastic_assignment import ( + binomial_share_tolerance, + stable_identity_uniforms, +) +from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime import property_components as components from microcosm.build.uk_runtime.hmrc_property_rental import ( HMRC_PROPERTY_RENTAL_EXPENSE_TYPES, HMRC_PROPERTY_RENTAL_RECEIPTS_BAND_LOWER_BOUNDS, @@ -152,6 +161,30 @@ def test_receipts_bands_must_be_table_13s(self) -> None: with pytest.raises(PropertyComponentsError, match="Table 13"): ReceiptsAllocationSpec.from_parameters(parameters) + def test_the_walk_declares_a_tie_salt_of_its_own(self) -> None: + # The receipts walk keys its tie-break on person ids under walk_seed + # and its own salt (microcosm#931): empty or missing is refused, and so + # is a salt the finance-cost draws key the same person ids on. + parameters = _parameters(ALLOCATE_PROPERTY_RENTAL_INCOME_KIND) + finance = FinanceCostModelSpec.from_parameters( + _parameters(IMPUTE_FRS_PROPERTY_FINANCE_COSTS_KIND) + ) + in_use = (finance.incidence_salt, finance.amount_salt) + spec = ReceiptsAllocationSpec.from_parameters(parameters, salts_in_use=in_use) + assert spec.tie_salt == "allocate_property_rental_income:tie" + assert spec.walk_seed == 0 + for bad in ("", None): + with pytest.raises(PropertyComponentsError, match="tie_salt"): + ReceiptsAllocationSpec.from_parameters({**parameters, "tie_salt": bad}) + without = {key: value for key, value in parameters.items() if key != "tie_salt"} + with pytest.raises(PropertyComponentsError, match="tie_salt"): + ReceiptsAllocationSpec.from_parameters(without) + for salt in in_use: + with pytest.raises(PropertyComponentsError, match="tie_salt"): + ReceiptsAllocationSpec.from_parameters( + {**parameters, "tie_salt": salt}, salts_in_use=in_use + ) + class TestFRSConcept: def test_the_reference_persons_sublet_rent_leaves_the_rental_profit(self) -> None: @@ -442,6 +475,116 @@ def test_receipts_do_not_depend_on_row_order(self) -> None: second, _ = self._allocate(ids[order], profit[order], weights[order]) np.testing.assert_allclose(first.receipts[order], second.receipts) + @staticmethod + def _by_id(monkeypatch): + """The walk's order before microcosm#931: ties by person id alone.""" + + monkeypatch.setattr( + components, + "keyed_tie_order", + lambda primary, ids, **_: np.lexsort((np.asarray(ids), primary)), + ) + + def test_distinct_profits_take_the_receipts_of_the_id_order( + self, monkeypatch + ) -> None: + # The tie-break only reorders exact ties, so on distinct profits + # every receipt and every cell of the receipt is the id order's bit + # for bit; only the declared order's wording moves (microcosm#931). + ids, profit, weights = _landlords() + keyed, _ = self._allocate(ids, profit, weights, receipts_per_profit=2.0) + self._by_id(monkeypatch) + by_id, _ = self._allocate(ids, profit, weights, receipts_per_profit=2.0) + + np.testing.assert_array_equal(keyed.receipts, by_id.receipts) + assert {**keyed.receipt, "order": None} == {**by_id.receipt, "order": None} + assert keyed.receipt["order"] == ( + "profit descending, landlords tied on profit in ascending order of a " + "uniform keyed on person_id under walk_seed and tie_salt, then " + "person_id ascending" + ) + + def test_tied_landlords_take_their_receipts_in_keyed_order( + self, monkeypatch + ) -> None: + # Six landlords of equal weight tie on profit inside one bounded band: + # the walk places them in the order of their keyed uniforms, which is + # not their id order, so the first takes the highest receipt. Their + # receipts are the id order's, among themselves permuted, and every + # other landlord's is unchanged (microcosm#931). + ids, profit, weights = _landlords() + tied = np.flatnonzero(profit > 0.0)[100:106] + profit[tied] = float(np.median(profit[profit > 0.0])) + weights[tied] = 1_000.0 + spec = ReceiptsAllocationSpec.from_parameters( + _parameters(ALLOCATE_PROPERTY_RENTAL_INCOME_KIND) + ) + uniforms = stable_identity_uniforms( + ids[tied], seed=spec.walk_seed, salt=spec.tie_salt + ) + keyed_order = tied[np.lexsort((ids[tied], uniforms))] + assert keyed_order.tolist() != sorted(keyed_order.tolist()) + + keyed, _ = self._allocate(ids, profit, weights, receipts_per_profit=2.0) + receipts = keyed.receipts[keyed_order] + assert len(set(receipts.tolist())) == len(tied) + assert (np.diff(receipts) < 0.0).all() + self._by_id(monkeypatch) + by_id, _ = self._allocate(ids, profit, weights, receipts_per_profit=2.0) + np.testing.assert_array_equal( + np.sort(keyed.receipts[tied]), np.sort(by_id.receipts[tied]) + ) + others = np.setdiff1d(np.arange(len(ids)), tied) + np.testing.assert_array_equal(keyed.receipts[others], by_id.receipts[others]) + order = np.random.default_rng(5).permutation(len(ids)) + monkeypatch.undo() + again, _ = self._allocate( + ids[order], profit[order], weights[order], receipts_per_profit=2.0 + ) + np.testing.assert_allclose(keyed.receipts[order], again.receipts) + + def test_clone_zero_takes_the_top_receipt_of_tied_clones_at_its_share( + self, monkeypatch + ) -> None: + # Sixty source landlords, each as three geographic clones of equal + # weight tied on profit, among the synthetic landlords. Walked by id, + # clone 0 took the highest receipt of its family every time; keyed on + # each clone's own id, at about a third of the families, inside the + # four-sigma binomial band (design verifier P10; microcosm#931). + ids, profit, weights = _landlords() + families, clone_count = 60, 3 + sources = np.arange(1, families + 1, dtype=np.int64) * 7 + 900_000 + clone_ids = np.concatenate( + [ + sources, + *( + copy_id_table.mint(sources, "geographic_support", k) + for k in range(1, clone_count) + ), + ] + ) + ids = np.concatenate([ids, clone_ids]) + profit = np.concatenate( + [profit, np.tile(np.linspace(3_000.0, 9_000.0, families), clone_count)] + ) + weights = np.concatenate([weights, np.full(families * clone_count, 900.0)]) + clones = slice(len(ids) - families * clone_count, len(ids)) + + def tops(allocation) -> np.ndarray: + receipts = allocation.receipts[clones].reshape(clone_count, families) + assert all(len(set(column)) == clone_count for column in receipts.T) + return np.argmax(receipts, axis=0) + + keyed, _ = self._allocate(ids, profit, weights, receipts_per_profit=2.0) + share = float((tops(keyed) == 0).mean()) + band = binomial_share_tolerance( + target=1 / clone_count, rows=families, sigma=4.0 + ) + assert abs(share - 1 / clone_count) <= band + self._by_id(monkeypatch) + by_id, _ = self._allocate(ids, profit, weights, receipts_per_profit=2.0) + assert float((tops(by_id) == 0).mean()) == 1.0 + class TestStageTransform: def _frame(self): @@ -520,6 +663,41 @@ def test_the_stage_refuses_a_frame_off_the_declared_build_period(self) -> None: with pytest.raises(PropertyComponentsError, match="build period 2024"): transform(moved) + def test_the_stage_refuses_a_tie_salt_its_finance_costs_draw_on(self) -> None: + # The receipts walk and the finance-cost draws key the same person + # ids, so the stage refuses a tie salt equal to either draw's salt + # (microcosm#931). + from microcosm.build.uk_runtime.property_components import ( + UKPropertyComponentsStageTransform, + ) + + stage = _stage() + finance = FinanceCostModelSpec.from_parameters( + _parameters(IMPUTE_FRS_PROPERTY_FINANCE_COSTS_KIND) + ) + for salt in (finance.incidence_salt, finance.amount_salt): + operations = tuple( + SourceOperationSpec( + kind=op.kind, + parameters=( + {**op.parameters, "tie_salt": salt} + if op.kind == ALLOCATE_PROPERTY_RENTAL_INCOME_KIND + else dict(op.parameters) + ), + ) + for op in stage.operations + ) + transform = UKPropertyComponentsStageTransform( + stage=replace(stage, operations=operations), + uprating_factors={ + "property_income": 1.0, + "property_finance_costs": 1.0, + }, + facts=_facts(landlords=1_000.0, receipts=1_000_000.0), + ) + with pytest.raises(PropertyComponentsError, match="tie_salt"): + transform(self._frame()) + def test_the_stage_adds_frs_costs_back_and_gives_every_landlord_receipts( self, tmp_path ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_localisation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_localisation.py index 568111495c..36c7e574bf 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_localisation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_localisation.py @@ -151,6 +151,7 @@ def test_support_copies_keep_the_stratum_mass_at_the_selected_share() -> None: # top five per cent and copies each six ways; the pool's uniforms must # leave exactly that share of the owner mass at or above the stratum. from microcosm.build.uk_runtime.high_value_home_support import ( + high_value_support_tie_break, select_high_value_support_households, ) @@ -167,6 +168,7 @@ def test_support_copies_keep_the_stratum_mass_at_the_selected_share() -> None: tenure_type=tenure, value=values, stratum_lower=0.95, + **dict(zip(("seed", "salt"), high_value_support_tie_break(), strict=True)), ) assert selected.sum() == pytest.approx(0.05 * n, abs=2) copies = 6 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py index 5ce6d4a1d0..eb3d262e35 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py @@ -1178,9 +1178,22 @@ def test_e8_declared_seed_lockstep(self) -> None: stages = load_country_spec("uk").sources.stage_map() assert stages["cgt_incidence_clone"].operations[1].parameters["seed"] == 0 - # The support split is deterministic: no seed, no salt (microcosm#1045). - assert "seed" not in stages["cgt_support_split"].operations[0].parameters - assert "salt" not in stages["cgt_support_split"].operations[0].parameters + # The support splits key their walks' tie-breaks under a seed and salt + # of their own, and the receipts walk under its walk seed and a tie + # salt (microcosm#931); before, the splits declared neither. + for stage, salt in ( + ("cgt_support_split", "cgt_support_split:wealth_tie"), + ("high_value_home_support", "high_value_home_support:value_tie"), + ): + parameters = stages[stage].operations[0].parameters + assert (parameters["seed"], parameters["salt"]) == (0, salt) + (receipts,) = [ + operation + for operation in stages["property_components"].operations + if operation.kind == "allocate_property_rental_income" + ] + assert receipts.parameters["walk_seed"] == 0 + assert receipts.parameters["tie_salt"] == "allocate_property_rental_income:tie" assert ( stages["hmrc_cgt_gains_spine"].operations[4].parameters["seed_base"] == 552 ) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json index a69be24ef5..9a5f9514c4 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/cgt_asset_type_facts.json @@ -2,70 +2,70 @@ "resource": "synthetic-cgt-asset-type.json", "resource_sha256": "synthetic", "source_commit": "synthetic", - "table4_all_gains": 6698701.203441119, + "table4_all_gains": 6699047.714667796, "table4_all_tax": 0.0, - "table4_all_taxpayers": 17.24436988246394, + "table4_all_taxpayers": 17.283592424982253, "table4_bands": [ { - "gains": 6998.611130126762, + "gains": 7081.25759032864, "lower_bound": 0, "tax": null, - "taxpayers": 1.8896108992721439, + "taxpayers": 1.909271876789354, "upper_bound": 10000 }, { - "gains": 37848.09165770648, + "gains": 36789.84760947093, "lower_bound": 10000, "tax": null, - "taxpayers": 2.3468115638209706, + "taxpayers": 2.2800823776945522, "upper_bound": 25000 }, { - "gains": 64412.5306127065, + "gains": 67268.60963759456, "lower_bound": 25000, "tax": null, - "taxpayers": 1.887044121541452, + "taxpayers": 1.96489044310873, "upper_bound": 50000 }, { - "gains": 106965.85772030169, + "gains": 107595.76161964415, "lower_bound": 50000, "tax": null, - "taxpayers": 1.5589961305052387, + "taxpayers": 1.565997699394142, "upper_bound": 100000 }, { - "gains": 247212.82520699556, + "gains": 249116.24079997113, "lower_bound": 100000, "tax": null, - "taxpayers": 1.5695959119641265, + "taxpayers": 1.5808559389601877, "upper_bound": 250000 }, { - "gains": 593976.5902481461, + "gains": 586673.5220109124, "lower_bound": 250000, "tax": null, - "taxpayers": 1.810480713045304, + "taxpayers": 1.7874674268214763, "upper_bound": 500000 }, { - "gains": 1466550.0712794645, + "gains": 1491792.5722846503, "lower_bound": 500000, "tax": null, - "taxpayers": 2.007093916729033, + "taxpayers": 2.0422967590985848, "upper_bound": 1000000 }, { - "gains": 4174736.6255856715, + "gains": 4152729.903115224, "lower_bound": 1000000, "tax": null, - "taxpayers": 4.1747366255856715, + "taxpayers": 4.152729903115224, "upper_bound": null } ], - "table4_individuals_gains": 6698701.203441119, + "table4_individuals_gains": 6699047.714667796, "table4_individuals_tax": 0.0, - "table4_individuals_taxpayers": 17.24436988246394, + "table4_individuals_taxpayers": 17.283592424982253, "table4_trusts_gains": 0.0, "table4_trusts_tax": 0.0, "table7_total_disposals": 2743095.2380952383, @@ -114,12 +114,12 @@ "proceeds": 9000000000.0 } ], - "table8a_disposals_total": 32.51795463550343, - "table8a_gains_total": 37770677.36531601, - "table8a_tax_total": 7554135.473063203, - "table8a_taxpayers_total": 29.56177694136675, - "table8b_all_gains": 37770677.36531601, - "table8b_all_taxpayers": 29.56177694136675, - "table8b_individuals_gains": 37770677.36531601, - "table8b_individuals_taxpayers": 29.56177694136675 + "table8a_disposals_total": 32.59191714425224, + "table8a_gains_total": 37717691.09595246, + "table8a_tax_total": 7543538.219190492, + "table8a_taxpayers_total": 29.629015585683852, + "table8b_all_gains": 37717691.09595246, + "table8b_all_taxpayers": 29.629015585683852, + "table8b_individuals_gains": 37717691.09595246, + "table8b_individuals_taxpayers": 29.629015585683852 } diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 85de0d0722..dc147ab1a0 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -563,7 +563,7 @@ ], "grain": "household", "nonnegative_outputs": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned and no draw is made; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", "operations": [ { "adult_minimum_age": 16, @@ -613,11 +613,13 @@ "minimum_gain_band_lower": 250000, "published_top_band_taxpayers": 58000, "reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", - "selection": "within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", + "salt": "cgt_support_split:wealth_tie", + "seed": 0, + "selection": "within each income band walk households in investable wealth descending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "support_mass_rule": "clone_split_factor x headroom x published taxpayers with gains at or above minimum_gain_band_lower in the income column", "suppressed_cells": "published counts withheld as fewer than 1,000 count as zero and are counted in the receipt", "taxpayer_concept": "hmrc.cgt_taxpayers_individuals", - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": "every column copied unchanged; the selection's tie-break is the stage's only draw", "wealth_measure": "household sum of investable_wealth_columns; a missing or non-finite value refuses the stage", "weight_kind_out": "importance" } @@ -3080,7 +3082,7 @@ "artifacts": [], "grain": "household", "nonnegative_outputs": [], - "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned and no draw is made here.", + "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned here; the one draw is the stratum's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders owners tied on value, so no copy or clone ordinal ranks highest on every tie.", "operations": [ { "conservation": "exact_total", @@ -3093,9 +3095,11 @@ "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", "kind": "split_top_value_owner_households", "reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", + "salt": "high_value_home_support:value_tie", + "seed": 0, "stratum_lower": 0.95, - "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", - "values": "every column copied unchanged; no draw, no seed, no salt", + "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", + "values": "every column copied unchanged; the stratum's tie-break is the stage's only draw", "weight_kind_out": "importance" } ], @@ -3175,7 +3179,7 @@ ], "population": "liable gainers whose arm is not residential (capital_gains_residential_property == 0, cgt_residential_split)", "qualifying_amount": "below the lifetime limit a claimant's whole net gain qualifies; in the open top band the qualifying gain is the lifetime limit", - "realization": "weighted systematic sampling within each band in ascending (gain, person_id) order with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare", + "realization": "weighted systematic sampling within each band in ascending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under seed and salted assign_badr_qualifying_gains:tie:, then person_id ascending, with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare", "reported_not_fitted": "tax at the relief rate, with the annual exempt amount set against the main-rate gains first as the engine allocates it, against Table 4.1's tax column", "seed": 555, "seed_mixing": "seed combined with the build period", @@ -3208,7 +3212,7 @@ "none_semantics": "no positive net gains", "output_column": "capital_gains_asset_type", "population": "liable gainers not flagged residential", - "realization": "error diffusion per claimant status in descending (gain, person_id) order: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; deterministic, no seed", + "realization": "error diffusion per claimant status in descending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under the assign_badr_qualifying_gains seed and salted assign_main_asset_type:tie:, then person_id ascending: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; that tie-break is its only draw", "share_targets": "Table 7 2023-24 gains by asset type excluding residential land and buildings, as shares", "stock_signals": { "listed_shares": "household stocks-and-shares ISA above zero, or the person's dividend income above zero", @@ -3388,7 +3392,7 @@ ], "kind": "rank_preserving_allocation", "minimum_allocation_people": 1, - "ordering": "a rank key descending, person_id ascending on ties: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers", + "ordering": "a rank key descending, then the rank-tie uniform ascending, then person_id ascending: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers; the rank-tie uniform is one identity-keyed uniform per person, keyed on person_id under seed_base and salted hmrc_cgt_gains_spine:rank_tie:, which every walk, plan and the remainder read", "rounding": "systematic with one seeded offset per walk: a person joins the band whose cumulative boundary first covers (1 - offset) of their weight, so a band boundary smaller than a person's weight still receives that person on the share of walks it implies and the rounding is unbiased across the conditioning cells; each cell walk's signed rounding error is carried into the next cell's boundaries for the same gain band (error diffusion), so each gain band's walked total within an income band stays within one person's weight of its raked total instead of accumulating the overshoot of many cell walks", "shortfall_policy": "proportional scale-down inside a cell that holds less gainer mass than its raked target; the income band's net shortfall after every cell walk, apportioned to gain bands in proportion to their positive shortfalls, is walked over the income band's pooled unassigned gainers, scaled down proportionally when the pool is short, so cell-walk rounding overshoot is netted rather than compounded", "suppressed_cell_allocation": "count implied by the cell's published gains at the band-total mean", @@ -3415,8 +3419,8 @@ "deterministic": true, "extrapolation": "ext=0", "kind": "sub_aea_remainder", - "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping draws nothing", - "ordering": "existing gains ascending, person_id descending on ties (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key)", + "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping makes no draw of its own", + "ordering": "existing gains ascending, then the rank-tie uniform descending, then person_id descending (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key)", "policy": "gainers beyond the published taxpayer mass take amounts from the Advani-Summers within-band gains distribution restricted to (0, annual exempt amount], rank-preserving within each Advani-Summers total-income band", "quantile_points": [ 0.05, @@ -6692,12 +6696,13 @@ ], "kind": "allocate_property_rental_income", "maximum_top_band_multiplier": 5, - "order": "profit descending, person_id ascending", + "order": "profit descending, landlords tied on profit in ascending order of a uniform keyed on person_id under walk_seed and tie_salt, then person_id ascending", "output": "property_rental_income", "population": "persons with profit from property other than sub-letting above zero, on both channels, after the finance-cost add-back", "residuals": "landlords without profit (FRS loss-makers, whose loss counts as zero, and tape landlords at zero) carry no receipts although PRIS counts them; property_finance_costs_brought_forward stays unset", "resource": "hmrc_property_rental_income_facts.json", "sublet_receipts": "the household reference person's sub-letting rent adds to receipts with no expenses; Rent a Room receipts, which the engine's receipts concept excludes, until PolicyEngine/policyengine-uk#2002 is released", + "tie_salt": "allocate_property_rental_income:tie", "top_band": "max(100,000, k x profit) with k at least 1, solved by bisection so the band's weighted mean receipts equal the individuals' Table 2 receipts less the bounded bands' realised mean receipts at their individuals-basis counts, over the top band's count; where the floor at k = 1 already exceeds that mean the multiplier stays 1 and the overshoot per landlord is recorded", "walk": "top band first, each band filled with whole persons to its share of the individuals-basis landlords times the spine's landlord weight (the spine's total before calibration differs from PRIS's, and filling the published counts would push every profit rank into a higher band), at one seeded systematic offset (the CGT Table 3 walk's rule); the lowest band takes every remaining landlord", "walk_seed": 0, diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_purchase_targets.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_purchase_targets.json index 33c37856ec..3b3a7c5685 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_purchase_targets.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_purchase_targets.json @@ -26,8 +26,8 @@ ], "minimum_price": 1000.0, "property_type": "additional_residential", - "transactions": 16.930405602541164, - "value": 132441.7306215811 + "transactions": 16.93040560254116, + "value": 139103.76725163503 }, "derivation": {}, "non_residential": { @@ -58,6 +58,6 @@ "minimum_price": 1000.0, "property_type": "non_residential", "transactions": 8.346596362360707, - "value": 125282.69039109821 + "value": 130821.50948812743 } } diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_rental_facts.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_rental_facts.json index 700e6fda1a..d6de64cabb 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_rental_facts.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/property_rental_facts.json @@ -9,17 +9,17 @@ "services_including_wages": 67.81449346944132 }, "expenses": { - "all": 32757.922728603626, - "individual": 32430.343501317588, + "all": 32757.922728603622, + "individual": 32430.343501317584, "partnership": 327.5792272860381 }, "expenses_by_type": { - "legal_management_and_professional_fees": 3921.52398707888, + "legal_management_and_professional_fees": 3921.5239870788796, "non_residential_finance_costs": 1253.756467022815, "other_allowable_expenses": 4317.44708192819, - "rent_rates_and_insurance": 3591.588074704455, - "repairs_and_maintenance": 6042.540566628755, - "residential_finance_costs": 12085.08113325751, + "rent_rates_and_insurance": 3591.5880747044544, + "repairs_and_maintenance": 6042.540566628754, + "residential_finance_costs": 12085.081133257509, "services_including_wages": 1545.98541798302 }, "landlords": { @@ -28,8 +28,8 @@ "partnership": 1.3562898693888314 }, "receipts": { - "all": 52412.6763657658, - "individual": 51888.54960210814, + "all": 52412.676365765794, + "individual": 51888.549602108134, "partnership": 524.1267636576595 }, "receipts_bands": [ diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index a9ec1d0364..047ed6f476 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"ed46fd4d2dc5df11ea9fdd0e955d8a7e13c20be5bae275b7a0a2f209a653fc78","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"698f3e0716d0bf95371bfc0b389179ce02d9108c904aff4d6fc433ebe6a08402","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"0edd2cdc89dbbb4fc9ad5332db4484522904f3bcede684cb89b189e99eafc537","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"660d72066059c354477e197f3c10cd27b6d3b91d433e3dc06c5551724a0a46ba","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"5e1c540675d3582a2840bc38f37600a0f6ce552471e6a2165c7705a6edc60a80","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"6d8c9503ad786ca285b96914fe9c1d792c7dcf5389841f51bbe295e549283ef8","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"1e513a7c6644d48ee0952a6d3f3151df3c6a8dd2289bec0b355f40fbbd10475d","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"a874b548dbb238143710da5220fec99e1ba4b69fcabe6bfcbba45de970ac77be","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"1dd10a33849c0377b6e79f5e5077ebad741b8f13de44896f05e7322700d90e3e","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"a0a0d7d311f217d9c8149656efd4b7baf7ee11c2eca2a904de694253ccd171e7","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index f8308aa40f..5d753a86b9 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -837,7 +837,9 @@ def _e8_support_split( ) -> dict[str, object]: """Recompute the CGT support split from the folded pre-split frame. - The split (microcosm#1045) is deterministic - no draw, no seed - so it is + The split (microcosm#1045) is deterministic - its one draw, the selection + walk's tie-break, is keyed on each household's own id under the declared + seed and salt (microcosm#931), which the recompute passes too - so it is reconstructible from the artifact alone: every clone folded onto its original and every support copy onto its root gives the pre-split household table at the weights the stage saw, and the stage's own helpers @@ -913,6 +915,7 @@ def _e8_support_split( cgt_support_household_wealth, cgt_support_income_band, cgt_support_mass_by_income_band, + cgt_support_tie_break, geographic_clone_count, select_cgt_support_households, ) @@ -1048,6 +1051,9 @@ def _e8_support_split( # The geographic clone count the copies were counted on, read from the # pre-split table as the stage read it from its input (microcosm#931). clone_count = geographic_clone_count(pre_household) + # The walk's declared tie-break, so households tied on wealth fall in the + # order the stage walked them (microcosm#931). + tie_seed, tie_salt = cgt_support_tie_break() def recompute(person_t, household_t, weights_t): ids_t = ( @@ -1060,7 +1066,13 @@ def recompute(person_t, household_t, weights_t): person_t, household_ids=ids_t, parameters=parameters ) selected, band_rows = select_cgt_support_households( - ids_t, weights_t, band, wealth, support_by_band + ids_t, + weights_t, + band, + wealth, + support_by_band, + seed=tie_seed, + salt=tie_salt, ) copies = np.ones(len(ids_t), dtype="int64") copies[selected] = cgt_support_copy_counts( From 610123e34c71191c9089de7c49304e8e4c2a709b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 01:26:00 +0000 Subject: [PATCH 042/108] Give tied ISA balances one weighted mid-rank (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María's ruling 7 of 7 Nov. Half of each subscriber's ISA amount key at stage 53, and of each Lifetime ISA payer's payment key, is the person's balance rank. _weighted_rank sorted balances stably and gave each row its own mid-rank, so tied balances ranked by row position, although its docstring said ties share a rank. Ties are common: the copies and clone halves made after the WAS wealth draw carry their root's ISA totals, so a single-adult one ties its root, and QRF draws repeat a few atoms. A root sits before its copies, as clone 0 will before its clones, so it ranked lowest among them. The stage was not row-order invariant where balances tie: the design's synthetic probe, re-run, moves 372 of 400 amounts under a row permutation on five atoms at the parent and none now (none on distinct balances at either). - _weighted_rank(values, weights, ids) orders by (value, id). Every member of a tie group takes (F(x-) + F(x)) / 2 over the total weight; a singleton keeps (c - w / 2) / total, computed as before. Both callers pass the person ids, so the keyed uniform half of each key alone orders tied holders, exchangeably, and every rank is the same bit for bit under any row order. The module docstring's step 3 says so. - No seed, salt or manifest key. No declared text says anything about ties (within_band stays "uniform"), so nothing regenerates: source_stages.json and the coverage manifest check current, and the H2 fixture holds no stage 53 value or kernel hash. No gates.json note mentions the rank. The band walk and the LISA strata stay pooled. - The rank stays in isa_subscriptions, its only user, so no shared module moves and only stage 53's kernel hash does. What moves (synthetic counts only): - Nothing where no balance ties: on the three synthetic stage frames of distinct balances every output is bit for bit the parent's. - On the same frames with balances on five atoms, every tied holder's rank moved and no singleton's (the rows arrive in id order, as the spine's tables do). Amounts still move beyond the ties, since the band walk and the within-band order are pooled: 248 to 260 of the 363 to 376 tied subscribers and 329 to 344 of the 420 to 430 others moved amount, and 46 to 56 of the 52 to 68 Lifetime ISA payers, all tied, moved payment. No balance moved. - On those frames a row permutation moved 588 to 599 of the 792 to 796 subscribers' amounts and every payer's payment at the parent, and moves none now. - With three geographic clones of 2,000 holders on five atoms, clone 0 gets the least of its three in 33 (ISA amounts) and 32 (LISA payments) per cent of the holders, against 43 and 40 by row position. - Node keys: 1 of the 99 UK kernel hashes moves, stage 53's. The gated spine re-keys 3 of its 81 nodes (isa_subscriptions, trading_allowance_receipts and the transferred gate) and the in-graph build 35 of its 113, from isa_subscriptions on; the national line on a bound spine keeps its 6. On the spine only stage 53's two subscription columns and its LISA payments move; stage 54 reads none of them. Tests (engine-free, synthetic or code facts): - New: a tie group's members share its mid-rank, by hand and over 3,000 tie-heavy rows; on id-sorted rows singletons keep the old rank bit for bit, and without ties every row does; the ranks are unchanged under five row permutations, where the old rank moved nearly every tied row's; with three clones of 2,000 holders, clone 0's share of the least ISA amount and of the least LISA payment lies inside the four-sigma binomial band around a third, and the old rank, patched in, puts both outside. - test_reordering_the_rows_moves_no_draw gains a five-atom case, which checks the ties are there. It fails at the parent (27.5 per cent of cash_isa_subscriptions differ) and passes now; the distinct case passes at both. The clone test fails at the parent too. - 8 mutations of a scratch copy are each caught and the copy as is passes: the old rank; the group mid-rank on row-order sums; ties at F(x) or at F(x-); singletons by the group formula; either call site on the old rank; ties by descending id. - The ISA stage-health entries pass on the five-atom frames at the parent and now. - test_uk_isa_subscriptions (22), test_uk_isa_stage_health (17) and test_uk_graph (53) pass. Ruff is clean on the 2 edited Python files. Co-Authored-By: Claude Opus 5.5 --- .../931-isa-tied-balance-rank.fixed.md | 9 + .../build/uk_runtime/isa_subscriptions.py | 41 ++- .../uk/test_uk_isa_subscriptions.py | 253 +++++++++++++++++- 3 files changed, 287 insertions(+), 16 deletions(-) create mode 100644 changelog.d/931-isa-tied-balance-rank.fixed.md diff --git a/changelog.d/931-isa-tied-balance-rank.fixed.md b/changelog.d/931-isa-tied-balance-rank.fixed.md new file mode 100644 index 0000000000..3818fd0efb --- /dev/null +++ b/changelog.d/931-isa-tied-balance-rank.fixed.md @@ -0,0 +1,9 @@ +The UK ISA subscriptions stage (`isa_subscriptions`, stage 53) gives every holder tied on a balance one weighted mid-rank (#931; María's ruling 7 of 7 Nov). Half of each subscriber's ISA amount key, and of each Lifetime ISA payer's payment key, is the person's balance rank, blended with a keyed uniform. The rank sorted balances stably and gave each row its own mid-rank, so tied balances ranked by row position, although its docstring said ties share a rank. Ties are common: the copies and clone halves made after the WAS wealth draw carry their root's ISA totals, so a single-adult one ties its root, and QRF draws repeat a few atoms. A root sits before its copies in the table, so it ranked below them, as clone 0 would below its clones once the spine clones households at its start. The stage was also not invariant to row order where balances tie: on a synthetic walk of 400 holders with balances on five atoms, permuting the rows moved 372 amounts, and none with distinct balances. + +- **The rank.** `_weighted_rank(values, weights, ids)` orders the rows by (value, id). Every member of a tie group takes the group's mid-rank, `(F(x-) + F(x)) / 2` over the total weight. A value held by one row keeps its own, `(c - w / 2) / total`, computed as before. Both callers pass the person ids, so the keyed uniform alone orders tied holders, exchangeably, and each rank is the same bit for bit whatever the row order. +- **No seed, no declaration.** The rank draws nothing, and no declared text says anything about ties, so no seed, salt, manifest key or generated file changes. The band walk and the Lifetime ISA strata stay pooled. +- **What moves.** Nothing where no balance ties: on three synthetic frames of distinct balances every output is unchanged, bit for bit. Elsewhere: + - Tied holders' ranks move. A singleton's rank is unchanged bit for bit wherever the rows arrive in id order, as the spine's tables do, but its amount can still move, since the band walk and the within-band order are pooled over the holders a tie moves past. + - On three synthetic frames with balances on five atoms, every tied holder's rank moved; 248 to 260 of the 363 to 376 tied subscribers and 329 to 344 of the 420 to 430 other subscribers moved amount; and 46 to 56 of the 52 to 68 Lifetime ISA payers, all of them tied, moved payment. No balance moved. + - With three geographic clones per holder on five atoms, clone 0 now gets the least of its three in about a third of the holders, against about 43 per cent (ISA amounts) and 40 per cent (Lifetime ISA payments) before. + - Of the 99 UK kernel hashes only stage 53's moves. Node keys move from stage 53 on: 3 of the gated spine's 81 nodes and 35 of the in-graph build's 113. The national line on a bound spine keeps its keys. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py index 018a0f3778..c4debc1b8c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py @@ -27,7 +27,8 @@ cells of the same income band with room; nobody is made a holder. 3. **How much (2023-24).** Within each (type, income band) the subscribers walk Table 9.7's band masses from the top, ranked by a key that blends their balance - rank with a keyed uniform, so larger holders tend to pay in more. Amounts are + rank with a keyed uniform, so larger holders tend to pay in more; holders tied + on a balance share its rank, so the keyed uniform alone orders them. Amounts are spread evenly inside each band on weight-proportional strata; GBP 20,000 is a point mass. 4. **The split for people paying into both** (María's ruling, 28 October 2026). @@ -991,17 +992,38 @@ def _subscription_bands() -> list[tuple[float, float]]: SUBSCRIPTION_BANDS = _subscription_bands() -def _weighted_rank(values: np.ndarray, weights: np.ndarray) -> np.ndarray: - """Each value's weighted mid-rank in [0, 1] (ties share a rank).""" +def _weighted_rank( + values: np.ndarray, weights: np.ndarray, ids: np.ndarray +) -> np.ndarray: + """Each value's weighted mid-rank in [0, 1]; tied values share one rank. + + A value held by one row keeps its own mid-rank, the weight below it plus + half its own, over the total. Every row of a group tied on one value takes + the group's mid-rank, ``(F(x-) + F(x)) / 2``: the weight below the group + plus half the group's. The weights accumulate in (value, id) order, so the + ranks do not depend on row order, and the keyed uniform each caller blends + in orders tied rows exchangeably. Tied rows used to rank by row position, + so the copies and clones that carry their root's balance ranked by where + they sat in the table, clone 0 lowest (microcosm#931). + """ - order = np.argsort(values, kind="stable") - cumulative = np.cumsum(weights[order]) + order = np.lexsort((ids, values)) + ordered = values[order] + ordered_weights = weights[order] + cumulative = np.cumsum(ordered_weights) total = float(cumulative[-1]) if len(cumulative) else 0.0 ranks = np.empty(len(values), dtype=float) if total <= 0: ranks[:] = 0.5 return ranks - ranks[order] = (cumulative - weights[order] / 2.0) / total + # A singleton's mid-rank, computed as before so it is bit for bit what it + # was wherever the rows arrive in id order (the spine's tables do). + mid = cumulative - ordered_weights / 2.0 + first = np.searchsorted(ordered, ordered, side="left") + last = np.searchsorted(ordered, ordered, side="right") - 1 + below = np.concatenate(([0.0], cumulative))[first] + mid = np.where(last > first, (below + cumulative[last]) / 2.0, mid) + ranks[order] = mid / total return ranks @@ -1032,7 +1054,8 @@ def draw_band_amounts( person_ids[rows], seed=seed, salt=f"{salt}:rank" ) rank_key = ( - balance_rank_weight * _weighted_rank(balance[rows], row_weights) + balance_rank_weight + * _weighted_rank(balance[rows], row_weights, person_ids[rows]) + (1.0 - balance_rank_weight) * uniforms ) ranked = np.lexsort((person_ids[rows], -rank_key)) @@ -1290,7 +1313,9 @@ def draw_lifetime_isa_payments( key = ( spec.balance_rank_weight * _weighted_rank( - recipients["lifetime_isa_balance"].to_numpy(float)[rows], payer_weights + recipients["lifetime_isa_balance"].to_numpy(float)[rows], + payer_weights, + person_ids, ) + (1.0 - spec.balance_rank_weight) * uniforms ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py index 6248b19609..5a9aedb3ac 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py @@ -10,6 +10,8 @@ from microcosm.build.source_manifest import SourceOperationSpec from microcosm.build.stage_evidence import snapshot_stage_evidence +from microcosm.build.stochastic_assignment import binomial_share_tolerance +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime import isa_subscriptions as stage from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, @@ -53,7 +55,17 @@ def donor(spec): ) -def _population(seed: int = 1, households: int = 800): +#: Five balance atoms, as repeated QRF draws give them: holders tie on them. +BALANCE_ATOMS = (1_000.0, 5_000.0, 12_000.0, 20_000.0, 45_000.0) + + +def _population( + seed: int = 1, households: int = 800, atoms: tuple[float, ...] | None = None +): + """The synthetic population; with ``atoms``, every positive household ISA + total and every Lifetime ISA balance is one of them, drawn from a second + generator so the other draws stay as they are.""" + rng = np.random.default_rng(seed) household = pd.DataFrame( { @@ -104,6 +116,12 @@ def _population(seed: int = 1, households: int = 800): "lifetime_isa_balance": rng.uniform(100, 20_000, len(person)), } ) + if atoms is not None: + pick = np.random.default_rng(seed + 1_000) + for column in ("cash_isa", "stocks_and_shares_isa"): + held = household[column].to_numpy() > 0 + household[column] = np.where(held, pick.choice(atoms, households), 0.0) + recipients["lifetime_isa_balance"] = pick.choice(atoms, len(person)) return household, person, recipients @@ -290,14 +308,212 @@ def test_band_amounts_sit_inside_their_bands_with_an_exact_top(statistics): assert not ((amounts > 18_000) & (amounts < 20_000) & (amounts >= 19_999.999)).any() +def _row_position_rank(values: np.ndarray, weights: np.ndarray) -> np.ndarray: + """The balance rank before microcosm#931: tied values ranked in row order.""" + + order = np.argsort(values, kind="stable") + cumulative = np.cumsum(weights[order]) + total = float(cumulative[-1]) if len(cumulative) else 0.0 + ranks = np.empty(len(values), dtype=float) + if total <= 0: + ranks[:] = 0.5 + return ranks + ranks[order] = (cumulative - weights[order] / 2.0) / total + return ranks + + +def _tie_heavy(rng: np.random.Generator, n: int): + """Values half on the five atoms and half distinct, weights, unique ids.""" + + values = np.where( + rng.random(n) < 0.5, + rng.choice(BALANCE_ATOMS, n), + rng.uniform(100.0, 60_000.0, n), + ) + weights = rng.uniform(1.0, 50.0, n) + ids = rng.choice(10**9, n, replace=False).astype(np.int64) + return values, weights, ids + + +def test_tied_balances_share_one_weighted_mid_rank(): + # The three rows tied on 5 take the group's mid-rank: the weight below it + # (6) plus half the group's (6), over the total (12). The two singletons + # keep their own. By row position the tied rows ranked 6.5, 8.5 and 11 + # twelfths (microcosm#931). + values = np.asarray([5.0, 1.0, 5.0, 3.0, 5.0]) + weights = np.asarray([1.0, 2.0, 3.0, 4.0, 2.0]) + ids = np.asarray([10, 20, 30, 40, 50], dtype=np.int64) + np.testing.assert_array_equal( + stage._weighted_rank(values, weights, ids), + [9 / 12, 1 / 12, 9 / 12, 4 / 12, 9 / 12], + ) + np.testing.assert_array_equal( + _row_position_rank(values, weights)[[0, 2, 4]], [6.5 / 12, 8.5 / 12, 11 / 12] + ) + # On tie-heavy input every member of a group shares one rank, + # (F(x-) + F(x)) / 2 over the total, which for a singleton is its own. + values, weights, ids = _tie_heavy(np.random.default_rng(20261110), 3_000) + ranks = stage._weighted_rank(values, weights, ids) + total = weights.sum() + sizes = [] + for value in np.unique(values): + group = values == value + sizes.append(int(group.sum())) + expected = (weights[values < value].sum() + weights[group].sum() / 2) / total + assert len(set(ranks[group].tolist())) == 1 + assert ranks[group][0] == pytest.approx(expected, rel=0, abs=1e-12) + tie_groups = [size for size in sizes if size > 1] + assert len(tie_groups) == len(BALANCE_ATOMS) and min(tie_groups) > 200 + # No weight: every rank is the middle, as before. + assert (stage._weighted_rank(values, np.zeros_like(weights), ids) == 0.5).all() + + +def test_singleton_ranks_are_the_row_position_ranks_on_id_sorted_rows(): + # The spine's tables arrive in id order, so a balance nobody else holds + # keeps its rank bit for bit and only tied rows move, to their group's + # mid-rank (microcosm#931). + values, weights, ids = _tie_heavy(np.random.default_rng(1110), 3_000) + in_id_order = np.argsort(ids) + values, weights, ids = values[in_id_order], weights[in_id_order], ids[in_id_order] + tied = pd.Series(values).duplicated(keep=False).to_numpy() + assert 0.4 < tied.mean() < 0.6 + + ranks = stage._weighted_rank(values, weights, ids) + before = _row_position_rank(values, weights) + + np.testing.assert_array_equal(ranks[~tied], before[~tied]) + assert (ranks[tied] != before[tied]).mean() > 0.99 + # Without ties every rank is the row-position rank, bit for bit. + np.testing.assert_array_equal( + stage._weighted_rank(values[~tied], weights[~tied], ids[~tied]), + _row_position_rank(values[~tied], weights[~tied]), + ) + + +def test_the_ranks_do_not_depend_on_row_order(): + # The weights accumulate in (value, id) order, so each row's rank is the + # same bit for bit however the rows arrive; by row position, tied rows' + # ranks followed their rows (microcosm#931). + values, weights, ids = _tie_heavy(np.random.default_rng(11), 2_000) + tied = pd.Series(values).duplicated(keep=False).to_numpy() + ranks = pd.Series(stage._weighted_rank(values, weights, ids), index=ids) + before = pd.Series(_row_position_rank(values, weights), index=ids) + for seed in range(5): + rows = np.random.default_rng(seed).permutation(len(values)) + permuted = stage._weighted_rank(values[rows], weights[rows], ids[rows]) + np.testing.assert_array_equal(permuted, ranks.loc[ids[rows]].to_numpy()) + old = _row_position_rank(values[rows], weights[rows]) + followed = old != before.loc[ids[rows]].to_numpy() + assert followed[tied[rows]].mean() > 0.99 + + +def _clone_pool(*, sources: int, clone_count: int, seed: int = 931): + """``sources`` holders and their geographic clones as one walk's payers: + clone-major, as their ids sort, each clone carrying its source's balance + (on the five atoms; both draws rank it) and an equal share of its weight.""" + + rng = np.random.default_rng(seed) + roots = 10_000 + np.arange(sources, dtype=np.int64) + ids = np.concatenate( + [ + roots, + *( + copy_id_table.mint(roots, "geographic_support", k) + for k in range(1, clone_count) + ), + ] + ) + balance = np.tile(rng.choice(BALANCE_ATOMS, sources), clone_count) + weights = np.tile(rng.uniform(5.0, 30.0, sources) / clone_count, clone_count) + return pd.DataFrame( + { + "person_id": ids, + "weight": weights, + "age": np.full(len(ids), 30.0), + "holds_lifetime_isa": np.ones(len(ids), dtype=bool), + "lifetime_isa_balance": balance, + } + ) + + +def _clone_zero_least(amounts: np.ndarray, clone_count: int) -> tuple[float, int]: + """The share of sources whose clone 0 gets strictly the least of its + clones, over the sources whose clones' amounts all differ, and their count.""" + + by_clone = amounts.reshape(clone_count, -1) + distinct = (np.diff(np.sort(by_clone, axis=0), axis=0) > 0).all(axis=0) + least = (by_clone[0] < by_clone[1:]).all(axis=0) + return float(least[distinct].mean()), int(distinct.sum()) + + +def test_clone_zero_is_not_ranked_below_its_clones(spec, monkeypatch): + # Three geographic clones of 2,000 holders, clone-major as their ids sort, + # each carrying its source's balance on five atoms. Clone 0 gets the least + # ISA amount, and the least Lifetime ISA payment, of its three in about a + # third of the sources, inside the four-sigma binomial band. Ranked by row + # position, each clone 0 sat in the bottom third of its tie group and got + # the least in about 43 and 40 per cent of the sources, outside the band + # (microcosm#931). + clone_count = 3 + pool = _clone_pool(sources=2_000, clone_count=clone_count) + shares = np.full(len(stage.HMRC_ISA_SUBSCRIPTION_BAND_IDS), 1.0) + + def isa_amounts() -> np.ndarray: + return stage.draw_band_amounts( + np.arange(len(pool)), + weights=pool["weight"].to_numpy(), + balance=pool["lifetime_isa_balance"].to_numpy(), + person_ids=pool["person_id"].to_numpy(), + band_shares=shares, + balance_rank_weight=spec.balance_rank_weight, + seed=spec.amount_seed, + salt=spec.amount_salt, + key="clones", + ) + + def lifetime_isa_payments() -> np.ndarray: + payments, receipt = stage.draw_lifetime_isa_payments( + pool, + headroom=np.full(len(pool), 20_000.0), + target_gbp=float(pool["weight"].sum()) * 1_000.0, + spec=spec, + ) + assert receipt["saturated"] is None + return payments + + def clone_zero_least() -> list[tuple[float, float]]: + found = [] + for draw in (isa_amounts, lifetime_isa_payments): + share, count = _clone_zero_least(draw(), clone_count) + band = binomial_share_tolerance( + target=1 / clone_count, rows=count, sigma=4.0 + ) + found.append((share, band)) + return found + + for share, band in clone_zero_least(): + assert abs(share - 1 / clone_count) <= band + monkeypatch.setattr( + stage, + "_weighted_rank", + lambda values, weights, ids: _row_position_rank(values, weights), + ) + for share, band in clone_zero_least(): + assert share > 1 / clone_count + band + + class _UKEngine: country = "uk" -def _stage_frame(seed: int = 1, order: np.ndarray | None = None): +def _stage_frame( + seed: int = 1, + order: np.ndarray | None = None, + atoms: tuple[float, ...] | None = None, +): """The synthetic population as the 2024 frame the whole transform reads.""" - household, person, recipients = _population(seed) + household, person, recipients = _population(seed, atoms=atoms) holds = recipients["holds_lifetime_isa"].to_numpy() person = person.assign( person_benunit_id=person["person_household_id"].to_numpy() * 10, @@ -404,18 +620,39 @@ def test_the_stage_declares_no_fit_weight_records(statistics): assert "fit_weight_records" not in evidence -def test_reordering_the_rows_moves_no_draw(statistics): +@pytest.mark.parametrize( + "atoms", [None, BALANCE_ATOMS], ids=["distinct_balances", "five_balance_atoms"] +) +def test_reordering_the_rows_moves_no_draw(statistics, atoms): """Every draw is keyed on identity, so row order moves none. - Adding or removing rows can move others' amounts (the rakes, band walks and - growth factors read the whole population), so only order is invariant. + On five atoms most holders tie on their balance, as copies, clone halves + and repeated QRF draws do. Tied balances share one rank, so the keyed + uniform orders them, not their rows (microcosm#931); ranked by row + position, most of their amounts moved under this permutation. Adding or + removing rows can move others' amounts (the rakes, band walks and growth + factors read the whole population), so only order is invariant. """ columns = list(stage.UK_ISA_SUBSCRIPTIONS_OUTPUT_COLUMNS) transform = _transform(statistics) - first = transform(_stage_frame()).table("person") + first = transform(_stage_frame(atoms=atoms)).table("person") + if atoms is not None: + # The ties are there: most subscribers and Lifetime ISA payers share + # their balance with another. + paying = ( + first[stage.CASH_SUBSCRIPTIONS] + + first[stage.STOCKS_AND_SHARES_SUBSCRIPTIONS] + > 0 + ) + held = first[stage.CASH_BALANCE] + first[stage.STOCKS_AND_SHARES_BALANCE] + assert held[paying].duplicated(keep=False).mean() > 0.6 + lifetime = first.loc[ + first[stage.LIFETIME_ISA_PAYMENTS] > 0, stage.LIFETIME_ISA_BALANCE + ] + assert lifetime.duplicated(keep=False).mean() > 0.9 order = np.random.default_rng(7).permutation(len(first)) - second = transform(_stage_frame(order=order)).table("person") + second = transform(_stage_frame(order=order, atoms=atoms)).table("person") pd.testing.assert_frame_equal( first.set_index("person_id")[columns].sort_index(), second.set_index("person_id")[columns].sort_index(), From f59bd511b03a1be6dd83c18293bf07da6701f04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:30:31 +0000 Subject: [PATCH 043/108] Follow up the review of the keyed offsets, tie-break and ISA rank commits (#931) Three reviews of 65e50b99e, b8bd45336 and 0c850b1f3 found no must. This commit takes the five shoulds and the ten nits (two of them the same national-line point); none needs a ruling. The commits are not amended, so the corrections to their messages are listed below for the PR body. 0c850b1f3 needed no change. Tests: - Stage 44's label test also runs on a frame with a short second cell in the 37,700 income band, so that band's pooled walk runs too. Above band 0 a band's position and lower bound differ, so a pooled walk keyed on its position, or every pooled walk keyed on the lowest band, now fails it. - Stage 53: a new test records the uniform each of select_subscribers' cell walks and draw_band_amounts' walk reads, and each band jitter the amounts draw, and compares them with the pre-move expressions, computed from each call's own seed, salt and key. A call that keys or salts its draw otherwise now fails, where the helper test passed. - Stage 47: the keyed-ties test also hands each walk in turn the id order and asserts that the tied persons' BADR gains, then their asset types, move. A walk that draws the keyed order and walks another now fails it. No stage 47 clone-order guard is added: gains are plan amounts and rarely tie. - Stage 28: a new test on unequal-weight ties (half of 2,400 landlords on 60 round profits, at weights that differ) holds that, against the id order, no untied landlord changes band, while tied landlords cross and the bands' record counts move. Code: - Stage 47's band offset key refuses an edge that is not an integer, with a TypeError naming the type. _validate_table4 compares edges by value, so a roster read with float edges passed it and would have keyed every band's offset anew. A NumPy integer keys as the Python integer does. Both loaders give ints, so no value moves. - cgt_asset_type's module docstring: the residential flag is carried as weight by cgt_residential_split since microcosm#1063 and read here; the weighted systematic walk is described under the BADR claims, which use it; and the main type is assigned by error diffusion, with the keyed tie-break its only draw. - keyed_tie_order's docstring: the lowest copy ordinal always came first, not always won. Changelog fragments: - 931-cgt-keyed-offsets: the rake couples income bands; gainers cross the exempt amount; the national line's keys; ISA takes the adapter from cgt_imputation and the scalar from stochastic_assignment. - 931-keyed-tie-breaks: stage 28 with unequal-weight ties; stage 42 at one clone; which copy ordinal won (the last at stage 26); copy_rule names the cap of 64 candidates, not their row order; the national line's keys; five stage contract hashes. Corrections to the earlier messages, for the PR body (synthetic counts only): - 65e50b99e: - "No nesting is promised" also misses the rake. It seeds every cell from the frame's own gainer shares and fits the seed to age and region margins over all income bands, so a change in one cell's gainers moves the raked targets of cells in other income bands. On a synthetic frame, one added cell in the lowest band moved all 20 positive targets of a two-cell higher band, by up to 0.8 per cent. - On the carry frame 70 of the 1,512 gainers crossed the annual exempt amount (none on the other two frames), so the liable set moves, and with it stage 46's arms and row counts. - On a bound spine the national line re-keys 5 of its 6 nodes, all but the bound checkpoint: the national target, problem and gate kernels hash the country spec, which the commit edits. - ISA takes _KeyedOffset from cgt_imputation and the scalar from stochastic_assignment. - b8bd45336: - Stage 28: untied landlords keep their band. "Band counts unchanged" holds only where a tie group shares one weight, as on the commit's family frames. Where a tie group of unequal weights straddles a band boundary, tied landlords cross it, band counts and masses change, and the other landlords' receipts in the bands either side move. On three frames of 2,400 landlords, half on 60 round profits at weights that differ: 24 to 49 tied landlords changed band; 4 to 7 of the 11 bands' record counts changed; no untied landlord changed band; and 398 to 941 of about 1,200 untied landlords' receipts moved, by at most about 0.3 per cent. - Stage 42's zero at one clone comes from light synthetic families. At one clone the copies move where a band's crossing falls in a tie group whose members weigh above 60, as a high-value family's can. - The H2 fixture moved five stage contract hashes in uk_spine.json (cgt_support_split, high_value_home_support, property_components, hmrc_cgt_gains_spine, hmrc_cgt_asset_type_spine), not one. - The first copy or clone won every tie at stages 28, 42 and 44; at stage 26, which takes the top of an ascending order, the last did. - copy_rule names the cap of 64 candidates, not their order: tied weights take row order. - On a bound spine the national line re-keys 5 of its 6 nodes, as for 65e50b99e. What moves: node keys only. No value, declaration or generated file moves. - 50 of the 99 UK kernel hashes, those that hash stochastic_assignment or cgt_asset_type: 48 stage kernels, uk.create, and the national gates through the gate registry's reach. - Every node of the hermetic and gated spine (79 and 81) and of the in-graph build (113) re-keys. On a bound spine the national line re-keys 1 of its 6 nodes, the calibrated gates. - source_stages.json and the coverage manifest check current. No declaration changed, so the H2 fixture does not regenerate. No gates.json note goes stale. Tests (engine-free, synthetic or code facts): the four touched test files and test_stochastic_assignment pass (189 tests), as do test_uk_graph, test_uk_isa_stage_health, test_uk_cgt_residential_split and test_uk_identity_stability_receipts (183). Ruff is clean on the 6 edited Python files. Under scratch the unmutated copy passes, and 9 mutations are each caught: - the pooled walk keyed on the band's position, or on the lowest band; - the ISA jitter keyed :, the amounts walk salted :walks, and the cell walks salted :cells; - either stage 47 walk drawing the keyed order and walking the id order; - the band key without the integer check; - stage 28 keying near ties. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-cgt-keyed-offsets.changed.md | 6 +- changelog.d/931-keyed-tie-breaks.changed.md | 13 ++- .../microcosm/build/stochastic_assignment.py | 6 +- .../build/uk_runtime/cgt_asset_type.py | 87 ++++++++++++------- .../engine_free/uk/test_uk_cgt_asset_type.py | 61 +++++++++++++ .../engine_free/uk/test_uk_cgt_imputation.py | 33 +++++-- .../uk/test_uk_isa_subscriptions.py | 83 ++++++++++++++++++ .../uk/test_uk_property_components.py | 57 ++++++++++++ 8 files changed, 301 insertions(+), 45 deletions(-) diff --git a/changelog.d/931-cgt-keyed-offsets.changed.md b/changelog.d/931-cgt-keyed-offsets.changed.md index b1825b9c51..e79c2bc541 100644 --- a/changelog.d/931-cgt-keyed-offsets.changed.md +++ b/changelog.d/931-cgt-keyed-offsets.changed.md @@ -6,7 +6,7 @@ The CGT gains redraw (`hmrc_cgt_gains_spine`) and the BADR claims in the asset-t Bands and age groups are named by their lower bounds, region groups by their labels. The period moved from the Generator's seed into each salt, so the declarations still read "seed combined with the build period". -- **Shared scalar.** `microcosm.build.stochastic_assignment.stable_identity_uniform(key, *, seed, salt)` returns the draw `stable_identity_uniforms([key], ...)` gives for one label, and refuses a sequence. The duck-typed `random()` adapter the walks read (`_KeyedOffset`) moved from `isa_subscriptions` into `cgt_imputation`, beside `_walk_bands`; the ISA subscriptions take it and the shared scalar from there, and their walk offsets and band jitters are the uniforms they were, bit for bit. `property_components`' one receipts-walk offset stays as it is (ruling 6). +- **Shared scalar.** `microcosm.build.stochastic_assignment.stable_identity_uniform(key, *, seed, salt)` returns the draw `stable_identity_uniforms([key], ...)` gives for one label, and refuses a sequence. The duck-typed `random()` adapter the walks read (`_KeyedOffset`) moved from `isa_subscriptions` into `cgt_imputation`, beside `_walk_bands`; the ISA subscriptions take the adapter from `cgt_imputation` and the scalar from `stochastic_assignment`, and their walk offsets and band jitters are the uniforms they were, bit for bit. `property_components`' one receipts-walk offset stays as it is (ruling 6). - **Declarations.** The redraw's `seed_mixing` and the BADR `realization` texts say the above, and the sub-AEA remainder's `mapping` text no longer names a walk and plan draw order, since there is none. Each is identical in the code dictionary, `sources.yaml` and `source_stages.json`. `hmrc_cgt_gains_spine` and `hmrc_cgt_asset_type_spine` now compute through `stochastic_assignment`, so both bind it, as the derived binding rule requires. -- **No nesting promised.** The offsets and jitters are the same at every geographic clone count, and whichever other cells walk or skip. The walks themselves stay pooled: the rounding carry runs through an income band's cells, the pooled walk takes the band's net shortfall over its unassigned gainers, each plan stratifies members pooled from every cell, and each BADR band walks its whole pool. A change in one cell can still move the assignments and amounts of the other cells in its income band, and a plan's amounts whenever its members change. -- **What moves.** Every Table 3 walk's offset and every plan's jitter changes, so band membership near each walk boundary and the within-band quantiles move. On three synthetic frames, 9 of 1,002, 9 of 3,000 and 126 of 1,512 gainers changed gain band, and 338, 2,920 and 1,300 changed amount; no non-gainer moved. Every BADR band's offset changes, so each band's walk selects its claimants at new positions. On three synthetic frames of equal weights no claimant row of the old walk claims in the new one, while each frame's claimant count moved by one or two; the main asset types moved with the claims. ISA amounts do not move: on three synthetic frames every ISA output column is unchanged. The stage contracts of both stages, the H2 fixture's declarations and the 22 values of its synthetic asset-type facts (which the fixture tool sizes to the frame after the redraw), the coverage manifest's 25 source digests and `source_stages.json` are regenerated. Every node re-keys, since `graph_kernels` and `stochastic_assignment` changed and the spine's root binds `stochastic_assignment`; values move from stage 44 on, and `isa_subscriptions` re-keys through its `cgt_imputation` binding and its own module with no change in its output. +- **No nesting promised.** The offsets and jitters are the same at every geographic clone count, and whichever other cells walk or skip. The walks themselves stay pooled: the rounding carry runs through an income band's cells, the pooled walk takes the band's net shortfall over its unassigned gainers, each plan stratifies members pooled from every cell, and each BADR band walks its whole pool. A change in one cell can still move the assignments and amounts of the other cells in its income band, and a plan's amounts whenever its members change. Through the rake it can move cells in other income bands too: the rake seeds every cell from the frame's own (age group, region group | income band) gainer shares and fits the seed to the joint and to the Table 6 age and Table 5 region margins, which span every income band, so a change in one cell's gainers moves the raked targets of cells in other income bands, and with them possibly those cells' assignments and amounts. On a synthetic frame, adding one cell in the lowest income band moved all 20 positive raked targets of a two-cell higher band, by up to 0.8 per cent. +- **What moves.** Every Table 3 walk's offset and every plan's jitter changes, so band membership near each walk boundary and the within-band quantiles move. On three synthetic frames, 9 of 1,002, 9 of 3,000 and 126 of 1,512 gainers changed gain band, and 338, 2,920 and 1,300 changed amount; no non-gainer moved. On the third, 70 of the 1,512 gainers crossed the annual exempt amount, between a Table 3 plan and the sub-AEA remainder (none on the other two), so the liable set moves, and with it the households stage 46 splits into arms and the row count from there. Every BADR band's offset changes, so each band's walk selects its claimants at new positions. On three synthetic frames of equal weights no claimant row of the old walk claims in the new one, while each frame's claimant count moved by one or two; the main asset types moved with the claims. ISA amounts do not move: on three synthetic frames every ISA output column is unchanged. The stage contracts of both stages, the H2 fixture's declarations and the 22 values of its synthetic asset-type facts (which the fixture tool sizes to the frame after the redraw), the coverage manifest's 25 source digests and `source_stages.json` are regenerated. Every spine node re-keys, since `graph_kernels` and `stochastic_assignment` changed and the spine's root binds `stochastic_assignment`, and on a bound spine the national line re-keys 5 of its 6 nodes, all but the bound checkpoint, since the national target, problem and gate kernels hash the country spec, which this change edits; values move from stage 44 on, and `isa_subscriptions` re-keys through its `cgt_imputation` binding and its own module with no change in its output. diff --git a/changelog.d/931-keyed-tie-breaks.changed.md b/changelog.d/931-keyed-tie-breaks.changed.md index 1ef51f4682..43060ddd00 100644 --- a/changelog.d/931-keyed-tie-breaks.changed.md +++ b/changelog.d/931-keyed-tie-breaks.changed.md @@ -1,4 +1,4 @@ -Six walks on the UK spine that ordered exactly tied rows by id now order them by an identity-keyed uniform, then by id (#931; accepted as designed on 7 Nov). Ties broken by id alone put the lowest id first every time, and the copies and geographic clones of one household differ only in their copy ordinal, so the first copy or clone won every tie: on synthetic clone pairs the lower clone index ranked first in every tied pair, and in about half of them once ties are keyed. Values move only where an exact tie touches a cut or a position schedule. +Six walks on the UK spine that ordered exactly tied rows by id now order them by an identity-keyed uniform, then by id (#931; accepted as designed on 7 Nov). Ties broken by id alone put the lowest id first every time, and the copies and geographic clones of one household differ only in their copy ordinal, so one copy ordinal won every tie: the first copy or clone at stages 28, 42 and 44, and the last at stage 26, which takes the top of an ascending order. On synthetic clone pairs the lower clone index ranked first in every tied pair, and in about half of them once ties are keyed. Values move only where an exact tie touches a cut or a position schedule. - **Shared helper.** `microcosm.build.stochastic_assignment.keyed_tie_order(primary, ids, *, seed, salt)` returns `np.lexsort((ids, u, primary))`, with `u` the records' `keyed_tie_uniforms(ids, *, seed, salt)`: `stable_identity_uniforms` of `integer_record_keys(ids, label=salt)`. It is a secondary sort key, the form stage 27 already uses, so on distinct values the order is the id order's bit for bit; it draws the uniforms of tied records only. Each record is keyed on its own id, never on `spine_household_keys`, which strips the geographic branch and would tie the clones again, and the rank-inside-a-tie form is not used, since it moves records that have no tie off their cut. - **Stage 42** (`cgt_support_split`): households tied on investable wealth, seed 0, salt `cgt_support_split:wealth_tie`. High-value families tie exactly on the four wealth columns, since no stage from 26 to 42 writes them. @@ -7,6 +7,13 @@ Six walks on the UK spine that ordered exactly tied rows by id now order them by - **Stage 44** (`hmrc_cgt_gains_spine`): one uniform per person under the stage's seed (552), salted `hmrc_cgt_gains_spine:rank_tie:`, drawn once per stage call and read by every cell walk, pooled walk, plan and the sub-AEA remainder, which take it as `_ranked`'s keyword-only `tie` (without it, person_id alone breaks ties, as before). The plans and the remainder walk the rank ascending, so a tie's member with the higher uniform takes the lower stratum. - **Stage 47** (`hmrc_cgt_asset_type_spine`): the BADR walk and the main asset type walk, under the BADR seed (555), salted `assign_badr_qualifying_gains:tie:` and `assign_main_asset_type:tie:`. - **Declarations.** The two support splits declare their tie-break as new `seed` and `salt` keys and the receipts walk its `tie_salt`, in the code dictionaries, the strict schema (three operations), `sources.yaml` and `source_stages.json`; the selection, stratum rule, order, ordering, mapping, realization and values texts, the stage notes and the receipts walk's and asset types' receipt order texts say the above, and "no draw, no seed, no salt" and "deterministic, no seed" are gone. The build sidecar's declared seeds register the splits under their salts and the receipts walk's seed under its tie salt. -- **Stage 42's rule B.** The split's walk, `keeps_the_split`'s rerun of it and the identity tool's recompute all pass the declared pair (`cgt_support_tie_break()`); the selection takes no default, so neither the rerun nor the tool can walk ties in another order. The exact-total correction's candidate order (the heaviest households left whole first, tied weights in row order) is unchanged: it picks the row of a rounding-sized correction rather than drawing anything, and `copy_rule` states it. +- **Stage 42's rule B.** The split's walk, `keeps_the_split`'s rerun of it and the identity tool's recompute all pass the declared pair (`cgt_support_tie_break()`); the selection takes no default, so neither the rerun nor the tool can walk ties in another order. The exact-total correction's candidate order (the heaviest households left whole first, tied weights in row order) is unchanged: it picks the row of a rounding-sized correction rather than drawing anything. `copy_rule` names the cap of 64 candidates but not their order; tied weights take row order, so of a whole household's geographic clones, which weigh the same, the one in the earliest row is tried first. - **No nesting promised.** All six walks are pooled. The uniform is a function of each row's own id, so clone 0's equals the `K = 1` row's and every clone draws its own. -- **What moves.** Nothing where no exact tie touches a cut or a position schedule. On synthetic frames (counts only): at stage 42, a band's crossing household when it sits in a tie group, so with three clones per household 2 to 8 of about 850 households per frame changed copy count, and clone 0, which carried the most families in all six frames, carries the most in one; at one clone none moved, since the tied families' members there weigh too little to be copied; at stage 26, the tie group across a region's cut, 0 to 5 of 3,000 owners per frame; at stage 28, every landlord in a tie group can take a tied neighbour's receipts, about two thirds of the tied landlords' receipts moved and none outside the tie groups, the band counts unchanged; at stage 44, often, since the blended rank key ties gainers even on distinct inputs, so 43 to 99 of about 1,800 gainers moved amount where wealth varies and none where it does not; at stage 47, nothing, since plan amounts do not tie (three frames bit for bit). Every spine node re-keys, since `stochastic_assignment` changed and the spine's root binds it; values move from stage 26 on. `source_stages.json`, the coverage manifest's 25 source digests, the H2 fixture's declarations and stage contract hash, and the 39 values of the synthetic facts its tool sizes to the frames stages 28, 29 and 46 read are regenerated. +- **What moves.** Nothing where no exact tie touches a cut or a position schedule. Elsewhere, counted on synthetic frames: + - At stage 42, a band's crossing household when it sits in a tie group: with three clones per household, 2 to 8 of about 850 households per frame changed copy count, and clone 0, which carried the most families in all six frames, carries the most in one. At one clone none moved on those frames, whose synthetic tied families are light, most members at or under 60, the weight above which a selected household takes a second copy. Where a band's crossing falls in a tie group whose members weigh more, as a high-value family's can, the keyed order changes which members are copied at one clone too. + - At stage 26, the tie group across a region's cut: 0 to 5 of 3,000 owners per frame. + - At stage 28, no untied landlord changes band, since the same landlords sit above it in either order. Where a tie group shares one weight, as the copies of one household do, only its own landlords move: on three frames of six-row families, about two thirds of the tied landlords' receipts moved, none outside the tie groups, and the band counts held. Where a tie group of unequal weights straddles a band boundary, as different households reporting one profit do, the keyed order puts another tied landlord at the boundary. Tied landlords then cross it, the bands' record counts and masses change, and the receipts of the other landlords in the bands on either side move with them; inside a band whose members hold, a reordered tie group can still move later landlords' receipts by rounding. On three frames of 2,400 landlords, half of them on 60 round profits at weights that differ, 24 to 49 tied landlords changed band and the record counts of 4 to 7 of the 11 bands changed; no untied landlord changed band, and the receipts of 398 to 941 of the about 1,200 untied landlords moved, by at most about 0.3 per cent. + - At stage 44, often, since the blended rank key ties gainers even on distinct inputs: 43 to 99 of about 1,800 gainers moved amount where wealth varies, and none where it does not. + - At stage 47, nothing, since plan amounts do not tie (three frames bit for bit). + - Node keys: every spine node re-keys, since `stochastic_assignment` changed and the spine's root binds it; values move from stage 26 on. On a bound spine the national line re-keys 5 of its 6 nodes, all but the bound checkpoint, since the national target, problem and gate kernels hash the country spec, which this change edits. + - Regenerated: `source_stages.json`, the coverage manifest's 25 source digests, the H2 fixture's declarations and the five stages' contract hashes in `uk_spine.json`, and the 39 values of the synthetic facts its tool sizes to the frames stages 28, 29 and 46 read. diff --git a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py index 5be91c2d30..c76bb778e1 100644 --- a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py +++ b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py @@ -179,9 +179,9 @@ def keyed_tie_order( ``primary`` take the order of their uniforms, then of their ids. Ties broken on ids alone put the lowest id first every time, so where the copies or geographic clones of one household tie, the lowest copy - ordinal always won (microcosm#931). Each uniform is a function of its - record's id, so the order does not depend on the order of the rows. Pass - a negated key to walk it descending. + ordinal always came first (microcosm#931). Each uniform is a function + of its record's id, so the order does not depend on the order of the + rows. Pass a negated key to walk it descending. Only the records that share their ``primary`` value with another are drawn, since no other record's uniform can move it; NaNs tie with each diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py index cd55e7650e..f32270e81c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py @@ -34,32 +34,38 @@ weighted mean log gain of the liable population, ``h`` one for a gainer who shows residential stock (another residential property in the household, or property income) and ``s`` the declared stock log-odds - (microcosm#1014). ``(a, b)`` are solved by nested bisection so the - household-weighted expected count and gains of flagged persons equal the - individuals-basis Table 8a totals: residential gainers are many and - small relative to the whole (a fifth of the gains on a third of the - taxpayers), which the negative slope carries. Flags are then realised by - a weighted systematic walk in ascending gain order with one seeded - offset: the walk flags a person whenever the expected weight owed so far - reaches that person's weight, so the flagged weight tracks the expected - weight within one person's weight everywhere along the gains axis, and - both the realised count and the realised gains sit close to their - expectations on frames whose liable rows are few and heavy. The - Bernoulli sigma of each is reported as the envelope a gate can bound - with. A flagged person's whole net - gain is attributed to residential property (``capital_gains_residential_property``); - a taxpayer with both residential and other disposals is not split. + (microcosm#1014). ``(a, b)`` are solved by nested bisection + (``solve_residential_logistic``) so the household-weighted expected + count and gains of flagged persons equal the individuals-basis Table 8a + totals: residential gainers are many and small relative to the whole (a + fifth of the gains on a third of the taxpayers), which the negative + slope carries. The flag is not drawn. Since microcosm#1063 the + ``cgt_residential_split`` stage, which runs before this one and makes + that solve, carries ``p`` as weight: it splits every household carrying + liable gainers into arms at product weights, one per subset of + residential gainers, so at design weights the residential count and + gains are mass identities. This stage reads the arms: a person whose + ``capital_gains_residential_property`` is positive is a residential + gainer on that arm, and the whole net gain of a flagged person is + residential; a taxpayer with both residential and other disposals is + not split. 2. **BADR and Investors' Relief claims.** Among liable gainers not flagged residential, each Table 4.1 band draws its claimants from the persons whose net gain falls inside the band's range. Below the lifetime limit a claimant's whole net gain qualifies, so the band's claimants are solved like the residential flag: a logistic in centred log gains whose intercept matches the band's published claimant count and whose slope - matches its published qualifying gains, realised by the same weighted - systematic walk with one identity-keyed offset per band, keyed on its - bounds. Each band's logistic carries the same stock shift for a gainer - who shows business stock (household corporate wealth, or - self-employment income). The open top + matches its published qualifying gains. Each band's logistic carries the + same stock shift for a gainer who shows business stock (household + corporate wealth, or self-employment income). The claims are realised + by a weighted systematic walk (``_weighted_systematic_flags``) in + ascending gain order, persons tied on gain in the order of a uniform + keyed on their person_id, with one identity-keyed offset per band, + keyed on its bounds: the walk flags a person whenever the expected + weight owed so far reaches that person's weight, so the flagged weight + tracks the expected weight within one person's weight along the gains + axis. The Bernoulli sigma of each band's count and gains is reported as + the envelope a gate can bound with. The open top band holds claimants at the lifetime limit: its probability is flat in gains and varies only with the stock shift, its intercept matches the count, the count is the published gains over the limit (the published @@ -68,7 +74,7 @@ Table 4.1's tax column to within half a percent; Table 4.1 counts gains before losses while the frame carries net gains, and Investors' Relief is merged with BADR as in the engine's input. -3. **Main asset type.** Every liable gainer not flagged residential draws +3. **Main asset type.** Every liable gainer not flagged residential takes one of five Table 7 types from a size-tilted categorical: the type weight times a log-normal kernel in log gains centred on the type's Table 7 mean gain per disposal (listed shares near £5,500, unlisted @@ -76,15 +82,21 @@ declared stock floor where the type's stock signal is absent (listed shares: a stocks-and-shares ISA or dividends; unlisted shares: business stock; other financial assets: financial wealth). A BADR or - Investors' Relief claimant draws only from the three types the reliefs - apply to (unlisted shares, business land and buildings, other + Investors' Relief claimant takes only one of the three types the + reliefs apply to (unlisted shares, business land and buildings, other non-financial assets), so the type follows the claim. The type weights - are fitted, with that restriction, by multiplicative updates until the - household-weighted gains shares by type reproduce Table 7's - non-residential gains shares, and the fit refuses if it does not - converge; one seeded uniform per person realises the draw. The - composition by gain band is reported for the evidence note; nothing is - fitted to it. + are fitted once per claimant status, with that restriction, by + multiplicative updates until each status's household-weighted gains + shares by type reach its share targets, which together restore Table + 7's non-residential gains shares, and each fit refuses if it does not + converge. The types are then assigned by error diffusion + (``_diffuse_types``) in descending gain order, persons tied on gain in + the order of a uniform keyed on their person_id, the assignment's only + draw: each person adds its stake (weight times gain) times its type + probabilities to a running balance per type and takes the type it may + hold whose balance is largest, so the realised gains by type track the + fit. The composition by gain band is reported for the evidence note; + nothing is fitted to it. Non-gainers carry ``none``; positive gains at or below the annual exempt amount carry ``sub_aea`` (Table 8 describes taxpayers with a liability and @@ -1018,11 +1030,24 @@ def _badr_band_offset(band: HMRCCGTBADRBand, *, seed: int, time_period: int) -> Keyed on the band's lower and upper edges, salted with the build period, not on its position in the roster, so a facts revision that adds or drops a band leaves every other band's offset alone (microcosm#931). + The key formats the edges, so an edge read as the float ``50000.0`` + would key ``50000.0`` and draw the band's offset anew, though the roster + check, which compares edges by value, accepts it; an edge that is not an + integer is refused, as ``integer_record_keys`` refuses float record ids. """ - upper = "open" if band.upper_bound is None else band.upper_bound + edges = [band.lower_bound] + if band.upper_bound is not None: + edges.append(band.upper_bound) + for edge in edges: + if isinstance(edge, bool) or not isinstance(edge, (int, np.integer)): + raise TypeError( + "Table 4.1 band edges must be integers to key a band's walk " + f"offset, not {type(edge).__name__}." + ) + upper = "open" if band.upper_bound is None else int(band.upper_bound) return stable_identity_uniform( - f"{band.lower_bound}|{upper}", + f"{int(band.lower_bound)}|{upper}", seed=seed, salt=f"assign_badr_qualifying_gains:offset:{int(time_period)}", ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py index ffdcfb9159..fac15de42f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_asset_type.py @@ -779,6 +779,44 @@ def recording(probabilities, band_weights, order, offset): np.testing.assert_array_equal(after[outside], before[outside]) assert (before[outside] > 0.0).any() + def test_a_band_edge_that_is_not_an_integer_is_refused(self) -> None: + """A band's offset key formats its edges, so a roster whose edges + arrived as floats (50000.0) would key every band's offset anew, and + the roster check, which compares edges by value, accepts it; the key + refuses it instead (microcosm#931).""" + + gains = _synthetic_gains() + weights = np.full(gains.size, 60.0) + facts = _synthetic_facts(gains, weights) + floats = dataclasses.replace( + facts, + table4_bands=tuple( + dataclasses.replace( + band, + lower_bound=float(band.lower_bound), + upper_bound=( + None if band.upper_bound is None else float(band.upper_bound) + ), + ) + for band in facts.table4_bands + ), + ) + cgt_asset_type._validate_table4(floats) + with pytest.raises(TypeError, match="edges must be integers"): + self._run(gains, weights, facts=floats) + # A NumPy integer edge keys as the Python integer does. + for band in facts.table4_bands: + as_numpy = dataclasses.replace( + band, + lower_bound=np.int64(band.lower_bound), + upper_bound=( + None if band.upper_bound is None else np.int64(band.upper_bound) + ), + ) + assert cgt_asset_type._badr_band_offset( + as_numpy, seed=555, time_period=2024 + ) == cgt_asset_type._badr_band_offset(band, seed=555, time_period=2024) + def test_refuses_claims_the_eligible_types_cannot_hold(self) -> None: tiny = tuple( ( @@ -1170,6 +1208,29 @@ def test_both_walks_break_gain_ties_on_their_own_keyed_uniforms( reordered.add(salt) # Both walks met the forced tie and ordered it otherwise than by id. assert reordered == salts + # And each walks the order it drew: handing one of them the id order + # moves the tied persons' outcome, which a walk that drew the keyed + # order and walked another would leave as it is. + keyed = _assign(frame, facts, PARAMETERS, BADR_PARAMETERS)[0].table("person") + tied = keyed["capital_gains"].to_numpy() == 15_000.0 + order = cgt_asset_type.keyed_tie_order + for walked, column in ( + ("assign_badr_qualifying_gains:tie", CGT_BADR_GAINS_COLUMN), + ("assign_main_asset_type:tie", CGT_ASSET_TYPE_COLUMN), + ): + + def by_id_here(primary, ids, *, seed, salt, walked=walked): + if salt.startswith(walked): + return np.lexsort((np.asarray(ids), np.asarray(primary, float))) + return order(primary, ids, seed=seed, salt=salt) + + with monkeypatch.context() as patch: + patch.setattr(cgt_asset_type, "keyed_tie_order", by_id_here) + result, _ = _assign(frame, facts, PARAMETERS, BADR_PARAMETERS) + by_id = result.table("person") + np.testing.assert_array_equal(by_id["person_id"], keyed["person_id"]) + moved = by_id[column].to_numpy() != keyed[column].to_numpy() + assert moved[tied].any(), walked _, reseeded = self._recorded(monkeypatch, frame, facts, badr_seed=999) assert {seed for _, seed, *_ in reseeded} == {999} declared = cgt_asset_type_operation_parameters() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py index 20bc317ae9..66f4a11f81 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_imputation.py @@ -1518,6 +1518,10 @@ def test_refuses_a_copy_count_below_one(self) -> None: ) #: A cell that walks before every other one in the lowest income band. _KEYED_EARLY_CELL = (20_000.0, 20, "WALES", 15) +#: A second cell in the 37,700 income band, short of its raked target, so +#: that band's pooled walk runs too: above band 0 a band's position and its +#: lower bound differ, so the label test sees which one the key names. +_KEYED_SHORT_HIGHER_CELL = (55_000.0, 70, "LONDON", 5) def _cells_frame(cells, *, zeroed=()) -> Frame: @@ -1661,12 +1665,31 @@ class TestKeyedWalkOffsets: the change leaves alone. """ - def test_each_walk_and_plan_reads_one_uniform_keyed_on_its_label(self) -> None: - _, draws = _labelled_draws(_cells_frame(_KEYED_CELLS)) + @pytest.mark.parametrize( + ("cells", "added", "pools"), + [ + (_KEYED_CELLS, set(), {"pool:0"}), + ( + (*_KEYED_CELLS, _KEYED_SHORT_HIGHER_CELL), + {"cell:37700:65:london"}, + {"pool:0", "pool:37700"}, + ), + ], + ids=["lowest_band_pools", "a_higher_band_pools_too"], + ) + def test_each_walk_and_plan_reads_one_uniform_keyed_on_its_label( + self, cells, added: set[str], pools: set[str] + ) -> None: + """A pooled walk is keyed on its income band's lower bound. In band 0 + that is also the band's position, so the second frame makes the + 37,700 band pool as well, where a key on the position (pool:1) or on + the lowest band (pool:0) reads another label's uniform.""" - cells = {label for label in draws if label.startswith("cell:")} - assert cells == set(_KEYED_CELL_LABELS) - assert {label for label in draws if label.startswith("pool:")} == {"pool:0"} + _, draws = _labelled_draws(_cells_frame(cells)) + + walked = {label for label in draws if label.startswith("cell:")} + assert walked == set(_KEYED_CELL_LABELS) | added + assert {label for label in draws if label.startswith("pool:")} == pools plans = {label for label in draws if label.startswith("plan:")} assert {_label_band(label) for label in plans} == {"0", "37700", "50000"} for label, (uniforms, _) in draws.items(): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py index 5a9aedb3ac..a3279637d6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_isa_subscriptions.py @@ -694,6 +694,89 @@ def test_the_walk_offsets_and_jitters_are_the_uniforms_they_were(spec): assert jitter == before(["income_3:2"], seed=spec.amount_seed, salt=salt)[0] +def test_the_selection_and_amounts_draw_the_uniforms_they_drew( + spec, statistics, monkeypatch +): + """The uniforms the selection's cell walks and the amounts' walk read, + and each band jitter the amounts draw, are what the stage drew before + the keyed scalar moved into stochastic_assignment (microcosm#931): the + pre-move expressions, computed here from each call's own seed, salt and + key, so a call that keys or salts one of its draws otherwise fails.""" + + from microcosm.build import stochastic_assignment + + before = stochastic_assignment.stable_identity_uniforms + walk, scalar = stage._walk_bands, stage.stable_identity_uniform + walks: list[tuple[np.ndarray, float]] = [] + draws: list[tuple[str, str, float]] = [] + + def recording_walk(**kwargs): + walks.append((np.asarray(kwargs["ranked"]), kwargs["rng"].random())) + return walk(**kwargs) + + def recording_scalar(key, *, seed, salt): + value = scalar(key, seed=seed, salt=salt) + draws.append((key, salt, value)) + return value + + monkeypatch.setattr(stage, "_walk_bands", recording_walk) + monkeypatch.setattr(stage, "stable_identity_uniform", recording_scalar) + isa_type = stage.CASH_ONLY + + # The selection: one walk per cell with a target, its offset keyed on the + # cell, salted :cell. + _, _, recipients = _population() + eligible = recipients["age"].to_numpy() >= 18 + seed, salt = spec.select_seed, f"{spec.select_salt}:{isa_type}" + stage.select_subscribers( + recipients, + eligible, + stage._margins(statistics, isa_type), + seed=seed, + salt=salt, + ) + frame = recipients.loc[eligible, ["income_band", "age_band", "sex", "region"]] + assert len(walks) > 10 + for ranked, offset in walks: + (cell,) = set(frame.iloc[ranked].itertuples(index=False, name=None)) + key = "|".join(map(str, cell)) + assert offset == before([key], seed=seed, salt=f"{salt}:cell")[0], key + assert {used for _, used, _ in draws} == {f"{salt}:cell"} + + # The amounts: one walk keyed on the call's key, salted :walk, and one + # jitter per drawn band, keyed :, salted :jitter. + walks.clear() + draws.clear() + rng = np.random.default_rng(3) + rows = 400 + seed, salt = spec.amount_seed, f"{spec.amount_salt}:{isa_type}" + key = f"{isa_type}|{HMRC_ISA_INCOME_BAND_IDS[3]}|base" + stage.draw_band_amounts( + np.arange(rows), + weights=rng.uniform(1.0, 50.0, rows), + balance=rng.uniform(100.0, 60_000.0, rows), + person_ids=np.arange(1, rows + 1, dtype=np.int64) * 1_000, + band_shares=np.full(len(stage.HMRC_ISA_SUBSCRIPTION_BAND_IDS), 1.0), + balance_rank_weight=spec.balance_rank_weight, + seed=seed, + salt=salt, + key=key, + ) + assert [offset for _, offset in walks] == [ + before([key], seed=seed, salt=f"{salt}:walk")[0] + ] + jitters = {label: value for label, used, value in draws if used == f"{salt}:jitter"} + drawn_bands = [ + band + for band, (lower, upper) in enumerate(stage.SUBSCRIPTION_BANDS) + if lower != upper + ] + assert sorted(jitters) == sorted(f"{key}:{band}" for band in drawn_bands) + for label, value in jitters.items(): + assert value == before([label], seed=seed, salt=f"{salt}:jitter")[0], label + assert {used for _, used, _ in draws} == {f"{salt}:walk", f"{salt}:jitter"} + + def test_the_both_split_reports_saturation(): combined = np.full(10, 10_000.0) weights = np.ones(10) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py index 0e63d143d6..ee0afe81d6 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py @@ -585,6 +585,63 @@ def tops(allocation) -> np.ndarray: by_id, _ = self._allocate(ids, profit, weights, receipts_per_profit=2.0) assert float((tops(by_id) == 0).mean()) == 1.0 + def test_unequal_weight_ties_move_no_untied_landlord_across_a_band( + self, monkeypatch + ) -> None: + # Half the landlords tie on 60 round profits at weights that differ, + # as different households reporting one amount do. The keyed order + # puts another tied landlord at a band boundary than the id order did, + # so tied landlords cross it and the bands' record counts move. An + # untied landlord has the same landlords above it in either order, so + # it keeps its band, though its receipt can move with its band's + # members (microcosm#931). + rng = np.random.default_rng((2800, 0)) + n = 2_400 + atoms = np.round(np.exp(rng.normal(np.log(9_000.0), 1.0, 60)), -2) + profit = np.where( + rng.random(n) < 0.5, + atoms[rng.integers(0, atoms.size, n)], + np.exp(rng.normal(np.log(9_000.0), 1.0, n)), + ) + weights = rng.uniform(50.0, 900.0, n) + ids = np.arange(1, n + 1, dtype=np.int64) * 7 + landlord = profit > 0.0 + tied = pd.Series(profit).duplicated(keep=False).to_numpy() & landlord + assert tied.sum() > n // 3 + + def walked_bands() -> np.ndarray: + seen: dict[str, np.ndarray] = {} + order, walk = components.keyed_tie_order, components._walk_bands + + def recording_order(primary, ids, **kwargs): + seen["order"] = order(primary, ids, **kwargs) + return seen["order"] + + def recording_walk(ranked_weights, masses, offset): + seen["band"] = walk(ranked_weights, masses, offset) + return seen["band"] + + with monkeypatch.context() as patch: + patch.setattr(components, "keyed_tie_order", recording_order) + patch.setattr(components, "_walk_bands", recording_walk) + self._allocate(ids, profit, weights, receipts_per_profit=2.0) + band = np.full(n, -1) + band[np.flatnonzero(landlord)[seen["order"]]] = seen["band"] + return band + + keyed = walked_bands() + self._by_id(monkeypatch) + by_id = walked_bands() + + crossed = keyed != by_id + assert not crossed[landlord & ~tied].any() + assert crossed[tied].sum() > 10 + bands = len(HMRC_PROPERTY_RENTAL_RECEIPTS_BAND_LOWER_BOUNDS) + assert ( + np.bincount(keyed[landlord], minlength=bands) + != np.bincount(by_id[landlord], minlength=bands) + ).any() + class TestStageTransform: def _frame(self): From b4868fda2acbba4e3cebd1958514e99856d6d0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 04:27:03 +0000 Subject: [PATCH 044/108] Extract the derived binding rule into a shared test module (#931) The derived binding rule's walk lived in test_uk_graph.py. Holding the transport kernels to it needs the same walk under the transport's own tables, and country-neutral machinery belongs in one shared module, with each country's details passed in as a declaration. This commit moves the walk and its controls as they were. It changes no UK table, walk or key. - test_support/microcosm_build/derived_binding_rule.py holds the machinery. A country declares its tables in a RuleTables (core modules, refusal-only checks, kernel stops and declaration modules) and walks through a BindingRule built from it. The functions are test_uk_graph's, word for word, except that each reads its tables from the rule: - table-free: module_definitions, writes_nothing, statement_names, key_only_methods, merged, hashed_by and the rest; - the rule's methods: resolve, named, helper_kind, walk, helper_walk, kernel_body, kernel_walk and the engine-root walk; - four checks both countries need, taken out of UK test bodies: kernel_classes_defined (the scan for classes that declare a ref), findings (each walk's modules outside its key, its stops outside the key, the listed stops met and the core names outside the Frame), and the rule's refusal_only_violations and core_modules_off_run_graph_path. - The lazy-export tables and the in-place calls are facts about the first-party packages, so they are shared constants, and a rule's lazy exports default to them. - The generalisations the design flagged, as settled here: - declaration modules are optional and empty by default, so a walk then skips no module; - each country gives its own reasons for its core modules; what is shared is the check that each is on run_graph's path; - a rule caches each definition's kind, and the lazy exports, per instance, since both depend on its tables; - findings leaves the Frame's core names out for every country, as the UK did, since a country's Frame reason covers them. The graph-fact reasons of the UK's core names stay in its table here; the transport commit shares them. - tests/engine_free/shared/test_derived_binding_rule.py holds the five controls that moved: the refusal check, the helper rule, the kernel rule, the engine root and the lazy exports. Each builds a rule from its own scratch tables instead of monkeypatching the UK's. - test_uk_graph.py keeps its tables and its UK-only parts (the stage table, the engine root and the gate-battery phase walk), builds _RULE once from its tables, and binds the rule's methods to the names its code already used. Its functions and test bodies read as before, but for the four that now call the shared checks. What moves: nothing. No key, value, declaration or generated file. Tests: all code facts. - Parity: a scratch dump of every UK walk the rule computes is the same, byte for byte, at 8ddd0a63c and at this commit. It covers the 54 stage transforms and the engine methods each names, the 16 engine roots, the 54 merged stage walks, the bodies and walks of the 48 kernel classes behind the 99 UK kernels (with the default halts, and without the gate registry's stop), the 99 kernel rule walks with the 4 battery phases and their phase-code modules, the helper walks the UK tests take directly, the six refusal-only verdicts, the registry bindings and the stage-health dispatch, the spine-build installs and the 99 UK kernel hashes. - test_uk_graph (48 tests), test_derived_binding_rule (5), test_uk_target_compile_modules (53) and test_first_party_imports (6) pass. Ruff is clean, and the test plan verifies. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_derived_binding_rule.py | 452 ++++++ .../tests/engine_free/uk/test_uk_graph.py | 1412 +---------------- .../microcosm_build/derived_binding_rule.py | 1032 ++++++++++++ 3 files changed, 1556 insertions(+), 1340 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py create mode 100644 test_support/microcosm_build/derived_binding_rule.py diff --git a/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py b/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py new file mode 100644 index 0000000000..361a891d50 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py @@ -0,0 +1,452 @@ +"""Controls for the derived binding rule's walk (microcosm#931). + +``test_support.microcosm_build.derived_binding_rule`` holds the rule's +country-neutral machinery: each country declares its tables in a +``RuleTables`` and walks through a ``BindingRule`` built from them, as +test_uk_graph.py does for the UK. These controls pin, on scratch packages, +each way the walk follows what a root names, the kernel rule's choice of what +a kernel runs, the engine root, and the refusal check, each control with a +rule of its own tables; and that the packages' lazy exports are the ones the +walk reads. +""" + +from __future__ import annotations + +import ast +import importlib +import uuid +from pathlib import Path + +import microcosm +from test_support.microcosm_build.derived_binding_rule import ( + FIRST_PARTY_SCOPE, + LAZY_EXPORT_TABLES, + NO_LAZY_EXPORTS, + BindingRule, + RuleTables, + hashed_modules, + module_definitions, + writes_nothing, +) + + +def test_the_refusal_check_sees_each_way_a_function_writes() -> None: + """Control for ``writes_nothing``: a function that only refuses, or + returns its sole argument, passes; one that writes through any of these + shapes, or returns one of several arguments, does not.""" + refusing = { + "raise": "def f(frame):\n if frame is None:\n raise ValueError('x')\n", + "identity": ( + "def f(entity):\n" + " if entity not in {'person'}:\n" + " raise ValueError('x')\n" + " return entity\n" + ), + } + writing = { + "copyto": "def f(values, mask):\n np.copyto(values, 0.0, where=mask)\n", + "out": "def f(values):\n np.clip(values, 0, 1, out=values)\n", + "add_at": "def f(w, roots, copies):\n np.add.at(w, roots, w[copies])\n", + "shuffle": "def f(rng, values):\n rng.shuffle(values)\n", + "reverse": "def f(items):\n items.reverse()\n", + "augmented": "def f(values):\n values *= 2.0\n", + "subscript": "def f(values):\n values[0] = 1.0\n", + "inplace": "def f(table):\n table.drop(columns=['a'], inplace=True)\n", + "pick": "def f(left, right):\n return left\n", + "computed": "def f(values):\n return values + 1\n", + } + for label, source in {**refusing, **writing}.items(): + function = ast.parse(source).body[0] + assert writes_nothing(function) is (label in refusing), label + + +def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( + tmp_path, monkeypatch +) -> None: + """Control: a helper called by name, through a re-export, a lazy export, + by attribute on its module, through an alias or a computed constant, or + held only in an annotation (a record's field type included) is reached, + as are a numeric constant, a collection of strings and a function that + writes in place or only refuses but is not listed; a listed check, a + single string, a column name, a record of plain fields, a listed core + module, what only a core module names, and a helper the root never names + are not. The walk returns the names it stopped at and the core modules it + met, and follows a core module only when asked to.""" + scratch = f"helper_rule_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / scratch + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + # A package exporting its runtime module's names lazily, as + # microcosm.graph does. + (package / "lazily").mkdir() + (package / "lazily" / "__init__.py").write_text( + "from importlib import import_module\n\n" + "_EXPORTS = {'deferred': 'runtime', 'idle': 'runtime'}\n\n" + "def __getattr__(name):\n" + " if name not in _EXPORTS:\n" + " raise AttributeError(name)\n" + " return getattr(import_module(f'{__name__}.{_EXPORTS[name]}'), name)\n" + ) + (package / "lazily" / "runtime.py").write_text( + "def deferred(x):\n return x * 6\n\ndef idle(x):\n return x * 7\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + sources = { + "direct": "def draw(x):\n return x + 1\n", + "origin": "def total(x):\n return sum(x)\n", + "reexport": "from .origin import total\n", + "attribute": "def share(x):\n return x / 2\n", + "alias": "def _scale(x):\n return x * 3\n\nscale = _scale\n", + "computed": "def _build():\n return {'a': 1}\n\nTABLE = _build()\n", + "collaborator": ( + "class Contract:\n def rate(self, key):\n return 0.5\n" + ), + "fielded": ( + "from typing import NamedTuple\n\n" + "class Rate:\n def value(self):\n return 0.1\n\n" + "class Inputs(NamedTuple):\n rate: Rate\n" + ), + "numeric": "FLOOR = 10\n", + "codes": ( + "CODES = frozenset({'A', 'B'})\n" + "OWNER = {'OWNED': 'owner', 'RENTED': 'renter'}\n" + ), + "inplace": ( + "import numpy as np\n\n" + "def fold(weights, roots, copies):\n" + " np.add.at(weights, roots, weights[copies])\n" + ), + "guard": ( + "def guard(frame):\n" + " if frame is None:\n" + " raise ValueError('no frame')\n" + ), + "harmless": ( + "COLUMN = 'household_weight'\n\n" + "def check(frame):\n" + " if frame is None:\n" + " raise ValueError('no frame')\n\n" + "def column(entity):\n" + " return f'{entity}_support_channel'\n\n" + "class Record:\n" + " value: float\n" + ), + "unused": "def never(x):\n return x - 1\n", + "behind": "def kept(x):\n return x * 4\n", + "core": ( + "from .behind import kept\n\n" + "class Container:\n" + " def grow(self, x):\n" + " return kept(x)\n" + ), + "stage": ( + "from . import attribute\n" + "from .alias import scale\n" + "from .codes import CODES, OWNER\n" + "from .collaborator import Contract\n" + "from .computed import TABLE\n" + "from .core import Container\n" + "from .direct import draw\n" + "from .fielded import Inputs\n" + "from .guard import guard\n" + "from .harmless import COLUMN, Record, check, column\n" + "from .inplace import fold\n" + "from .lazily import deferred\n" + "from .numeric import FLOOR\n" + "from .reexport import total\n" + "from .unused import never\n\n" + "class Transform:\n" + " contract: Contract\n" + " inputs: Inputs\n\n" + " def __call__(self, frame):\n" + " check(frame)\n" + " guard(frame)\n" + " fold(frame, [0], [1])\n" + " record = Record(value=draw(1) + total([1]) + deferred(1))\n" + " rate = self.contract.rate(column('household'))\n" + " rate += self.inputs.rate.value()\n" + " owners = [OWNER[code] for code in CODES]\n" + " rate += Container().grow(rate)\n" + " return attribute.share(scale(rate)) + TABLE['a'] + FLOOR, (\n" + " record, COLUMN, owners\n" + " )\n\n" + "def helper():\n" + " return never(1)\n" + ), + } + for module, source in sources.items(): + (package / f"{module}.py").write_text(source) + prefix = f"microcosm.{scratch}." + rule = BindingRule( + RuleTables( + core_modules={f"{prefix}core": "control"}, + refusal_only={(f"{prefix}harmless", "check"): "control"}, + lazy_export_tables={**LAZY_EXPORT_TABLES, f"{prefix}lazily": "_EXPORTS"}, + ) + ) + reach, stops, core = rule.helper_walk((f"{prefix}stage", "Transform"), scope=prefix) + assert {module.removeprefix(prefix): names for module, names in reach.items()} == { + "stage": {"Transform"}, + "direct": {"draw"}, + "origin": {"total"}, + "lazily.runtime": {"deferred"}, + "attribute": {"share"}, + "alias": {"scale", "_scale"}, + "computed": {"TABLE", "_build"}, + "collaborator": {"Contract"}, + "fielded": {"Inputs", "Rate"}, + "numeric": {"FLOOR"}, + "codes": {"CODES", "OWNER"}, + "inplace": {"fold"}, + "guard": {"guard"}, + } + assert {(module.removeprefix(prefix), name) for module, name in stops} == { + ("harmless", "COLUMN"), + ("harmless", "Record"), + ("harmless", "column"), + } + assert core == {f"{prefix}core"} + followed = rule.helper_walk( + (f"{prefix}stage", "Transform"), scope=prefix, follow_core=True + ) + assert set(followed.reach) - set(reach) == {f"{prefix}core", f"{prefix}behind"} + assert followed.core == core + + +def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> None: + """Control: the kernel rule follows what a kernel's methods name, + through its bases by method resolution order (a base method it inherits, + and one it overrides only where it calls ``super()``), and leaves out its + implementation hash, a method only the hash calls, a listed refusal-only + method, a base method it overrides without ``super()`` and what lies + behind a listed kernel stop, which it reports with what named it. A + method the hash and the run both call stays. The recorder sees each + module the hash passes to ``source_hash``, in whichever module calls + it.""" + scratch = f"kernel_rule_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / scratch + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + sources = { + "computed": "def computed(x):\n return x + 1\n", + "inherited": "def base_only(x):\n return x * 2\n", + "assisted": "def kept(x):\n return x * 3\n", + "overridden": "def replaced(x):\n return x - 2\n", + "refused": "def check(x):\n return x < 0\n", + "keyonly": "def salt():\n return 'salt'\n", + "behind": "def beyond(x):\n return x * 5\n", + "resourced": "def fingerprint(x):\n return x * 7\n", + "shared": "def both(x):\n return x * 11\n", + "table": "from .behind import beyond\n\nTABLE = {'a': beyond}\n", + "keyed": ( + "from microcosm.graph import source_hash\n\n" + "from . import computed, keyonly\n\n" + "def key():\n return source_hash(computed, keyonly)\n" + ), + "base": ( + "from .assisted import kept\n" + "from .inherited import base_only\n" + "from .overridden import replaced\n\n" + "class Base:\n" + " def run(self, x):\n" + " return base_only(x)\n\n" + " def prepare(self, x):\n" + " return replaced(x)\n\n" + " def assist(self, x):\n" + " return kept(x)\n" + ), + "kernel": ( + "from .base import Base\n" + "from .computed import computed\n" + "from .keyed import key\n" + "from .keyonly import salt\n" + "from .refused import check\n" + "from .resourced import fingerprint\n" + "from .shared import both\n" + "from .table import TABLE\n\n" + "class Kernel(Base):\n" + " def implementation_hash(self):\n" + " return salt() + key() + self._resources(1) + self._both(1)\n\n" + " def _resources(self, x):\n" + " return str(fingerprint(x))\n\n" + " def _both(self, x):\n" + " return str(both(x))\n\n" + " def _admit(self, x):\n" + " if check(x):\n" + " raise ValueError('refused')\n\n" + " def prepare(self, x):\n" + " return x\n\n" + " def run(self, x):\n" + " self._admit(x)\n" + " self._both(x)\n" + " return computed(x) + super().run(x) + TABLE['a'](x)\n" + ), + } + for module, source in sources.items(): + (package / f"{module}.py").write_text(source) + prefix = f"microcosm.{scratch}." + rule = BindingRule( + RuleTables( + core_modules={}, + refusal_only={(f"{prefix}kernel", "Kernel._admit"): "control"}, + stops={(f"{prefix}table", "TABLE"): "control"}, + ) + ) + kernel_class = importlib.import_module(f"{prefix}kernel").Kernel + walk = rule.kernel_walk(kernel_class, scope=prefix) + assert { + module.removeprefix(prefix): names for module, names in walk.reach.items() + } == { + "kernel": {"Kernel"}, + "base": {"Base"}, + "computed": {"computed"}, + "inherited": {"base_only"}, + "assisted": {"kept"}, + "shared": {"both"}, + } + assert walk.met == { + (f"{prefix}table", "TABLE"): frozenset({(f"{prefix}kernel", "Kernel.run")}) + } + assert hashed_modules(kernel_class()) == {f"{prefix}computed", f"{prefix}keyonly"} + + +def test_the_engine_root_follows_the_methods_a_stage_calls( + tmp_path, monkeypatch +) -> None: + """Control for the engine root: the methods a stage names on an engine, + by attribute on any receiver or through ``getattr``, are followed, with + the constructor and the methods they call on ``self``; a method no stage + names is not, nor what only it computes through. A method a helper the + engine hands itself to calls on that argument is followed too, to a fixed + point (``engine_reach``), which one pass from the stage's methods + misses.""" + scratch = f"engine_root_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / scratch + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + sources = { + "tables": "def tables(frame):\n return frame\n", + "tree": "def tree(year):\n return year\n", + "vocabulary": "def vocabulary():\n return {'a': 1}\n", + "count": "def count(x):\n return int(x)\n", + "engine": ( + "from .count import count\n" + "from .tables import tables\n" + "from .tree import tree\n" + "from .vocabulary import vocabulary\n\n" + "class Engine:\n" + " def __init__(self, clones=1):\n" + " self._clones = count(clones)\n\n" + " def materialize(self, frame, names, period):\n" + " return self._dataset(frame)\n\n" + " def _dataset(self, frame):\n" + " return tables(frame)\n\n" + " def parameters(self, period):\n" + " return tree(period)\n\n" + " def concept_mapping(self):\n" + " return vocabulary()\n" + ), + "stage": ( + "class Transform:\n" + " def __call__(self, frame, model):\n" + " reader = getattr(model, 'parameters', None)\n" + " return model.materialize(frame, ['x'], 2024), reader\n" + ), + "delegating": ( + "from .engine import Engine\n\n" + "def _vocabulary_of(engine):\n" + " return engine.concept_mapping()\n\n" + "class Delegating(Engine):\n" + " def materialize(self, frame, names, period):\n" + " return _vocabulary_of(self)\n" + ), + } + for module, source in sources.items(): + (package / f"{module}.py").write_text(source) + prefix = f"microcosm.{scratch}." + rule = BindingRule(RuleTables(core_modules={})) + + def short(reach): + return {module.removeprefix(prefix): names for module, names in reach.items()} + + engine_class = importlib.import_module(f"{prefix}engine").Engine + walk = rule.helper_walk((f"{prefix}stage", "Transform"), scope=prefix) + methods = rule.engine_methods(engine_class, walk) + assert methods == {"materialize", "parameters"} + reach = rule.engine_walk(engine_class, methods, scope=prefix).reach + assert short(reach) == { + "engine": {"Engine"}, + "count": {"count"}, + "tables": {"tables"}, + "tree": {"tree"}, + } + assert rule.engine_reach(engine_class, walk, scope=prefix).reach == reach + delegating = importlib.import_module(f"{prefix}delegating").Delegating + once = rule.engine_walk( + delegating, rule.engine_methods(delegating, walk), scope=prefix + ).reach + assert "vocabulary" not in short(once) + assert short(rule.engine_reach(delegating, walk, scope=prefix).reach) == { + "delegating": {"Delegating", "_vocabulary_of"}, + "engine": {"Engine"}, + "count": {"count"}, + "tree": {"tree"}, + "vocabulary": {"vocabulary"}, + } + + +def test_every_lazy_export_resolves_to_the_object_python_returns() -> None: + """The walk follows a name a package exports lazily, through its module + ``__getattr__``, to the module that defines it, as Python imports it + (microcosm#931 ruling 8). Every name microcosm.graph exports that way + resolves to a definition holding the very object the package returns, + the table the walk reads is the one the package's ``__getattr__`` reads, + and a name the package imports eagerly resolves as before. Every + first-party module that defines a module ``__getattr__`` is listed, as a + lazy exporter (``LAZY_EXPORT_TABLES``) or as one that exports nothing + through it (``NO_LAZY_EXPORTS``), so a new one cannot hide names from the + walk.""" + from microcosm import graph + + rule = BindingRule(RuleTables(core_modules={})) + for module, table in LAZY_EXPORT_TABLES.items(): + getter = module_definitions(module, FIRST_PARTY_SCOPE).definitions[ + "__getattr__" + ] + assert table in { + sub.id for sub in ast.walk(getter) if isinstance(sub, ast.Name) + }, module + exports = getattr(graph, LAZY_EXPORT_TABLES[graph.__name__]) + assert {"load_source", "KernelContext", "keyed_uniform"} <= set(exports) + for name in exports: + found = rule.resolve(graph.__name__, name) + assert found is not None, name + module, defined = found + assert getattr(importlib.import_module(module), defined) is getattr( + graph, name + ), name + assert rule.resolve(graph.__name__, "compile_graph") == ( + "microcosm.graph.decl", + "compile_graph", + ) + getters = set() + for root in map(Path, microcosm.__path__): + for path in sorted(root.rglob("*.py")): + text = path.read_text(encoding="utf-8") + if "def __getattr__" not in text or not any( + isinstance(node, ast.FunctionDef) and node.name == "__getattr__" + for node in ast.parse(text).body + ): + continue + parts = path.relative_to(root.parent).with_suffix("").parts + getters.add(".".join(parts[:-1] if parts[-1] == "__init__" else parts)) + listed = set(LAZY_EXPORT_TABLES) | set(NO_LAZY_EXPORTS) + assert getters == listed, sorted(getters ^ listed) + # The modules that export nothing that way only raise from __getattr__. + for module in NO_LAZY_EXPORTS: + getter = module_definitions(module, FIRST_PARTY_SCOPE).definitions[ + "__getattr__" + ] + assert not any(isinstance(sub, ast.Return) for sub in ast.walk(getter)) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index d23cea14dd..8f5fadea70 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -9,6 +9,21 @@ from microcosm.graph import graph_schema, orrery_document from microcosm.graph.canonical import canonical_json +from test_support.microcosm_build.derived_binding_rule import ( + FIRST_PARTY_SCOPE, + BindingRule, + KernelWalk, + KeyedWalk, + RuleTables, + Walk, + class_named, + findings, + hashed_by, + hashed_modules, + kernel_classes_defined, + merged, + module_definitions, +) from test_support.microcosm_build.first_party_imports import hashed_module_names from test_support.microcosm_build.uk_graph import * @@ -874,7 +889,7 @@ def test_the_engine_stages_bind_what_the_injected_engine_computes_through() -> N is rooted at that class (``_ENGINE_CLASS``): it follows the methods the stage's code names on it, by attribute or through ``getattr``, with the constructor and every method they call on ``self``, at the first-party - scope with the stage rule's tables (``_engine_walk``), and the methods + scope with the stage rule's tables (``engine_walk``), and the methods its own reach names on any receiver outside the engine's classes, as a helper handed the engine might call, to a fixed point (``_engine_reach``). ``_stage_walks`` @@ -926,94 +941,6 @@ def test_the_engine_root_is_the_engine_the_builds_inject() -> None: assert _ENGINE_CLASS in built, (module, function) -def test_the_engine_root_follows_the_methods_a_stage_calls( - tmp_path, monkeypatch -) -> None: - """Control for the second root: the methods a stage names on an engine, - by attribute on any receiver or through ``getattr``, are followed, with - the constructor and the methods they call on ``self``; a method no stage - names is not, nor what only it computes through. A method a helper the - engine hands itself to calls on that argument is followed too, to a fixed - point (``_engine_reach``), which one pass from the stage's methods - misses.""" - import importlib - import uuid - - scratch = f"engine_root_{uuid.uuid4().hex[:8]}" - package = tmp_path / "microcosm" / scratch - package.mkdir(parents=True) - (package / "__init__.py").write_text("") - monkeypatch.syspath_prepend(str(tmp_path)) - sources = { - "tables": "def tables(frame):\n return frame\n", - "tree": "def tree(year):\n return year\n", - "vocabulary": "def vocabulary():\n return {'a': 1}\n", - "count": "def count(x):\n return int(x)\n", - "engine": ( - "from .count import count\n" - "from .tables import tables\n" - "from .tree import tree\n" - "from .vocabulary import vocabulary\n\n" - "class Engine:\n" - " def __init__(self, clones=1):\n" - " self._clones = count(clones)\n\n" - " def materialize(self, frame, names, period):\n" - " return self._dataset(frame)\n\n" - " def _dataset(self, frame):\n" - " return tables(frame)\n\n" - " def parameters(self, period):\n" - " return tree(period)\n\n" - " def concept_mapping(self):\n" - " return vocabulary()\n" - ), - "stage": ( - "class Transform:\n" - " def __call__(self, frame, model):\n" - " reader = getattr(model, 'parameters', None)\n" - " return model.materialize(frame, ['x'], 2024), reader\n" - ), - "delegating": ( - "from .engine import Engine\n\n" - "def _vocabulary_of(engine):\n" - " return engine.concept_mapping()\n\n" - "class Delegating(Engine):\n" - " def materialize(self, frame, names, period):\n" - " return _vocabulary_of(self)\n" - ), - } - for module, source in sources.items(): - (package / f"{module}.py").write_text(source) - prefix = f"microcosm.{scratch}." - - def short(reach): - return {module.removeprefix(prefix): names for module, names in reach.items()} - - engine_class = importlib.import_module(f"{prefix}engine").Engine - walk = _helper_walk((f"{prefix}stage", "Transform"), scope=prefix) - methods = _engine_methods(engine_class, walk) - assert methods == {"materialize", "parameters"} - reach = _engine_walk(engine_class, methods, scope=prefix).reach - assert short(reach) == { - "engine": {"Engine"}, - "count": {"count"}, - "tables": {"tables"}, - "tree": {"tree"}, - } - assert _engine_reach(engine_class, walk, scope=prefix).reach == reach - delegating = importlib.import_module(f"{prefix}delegating").Delegating - once = _engine_walk( - delegating, _engine_methods(delegating, walk), scope=prefix - ).reach - assert "vocabulary" not in short(once) - assert short(_engine_reach(delegating, walk, scope=prefix).reach) == { - "delegating": {"Delegating", "_vocabulary_of"}, - "engine": {"Engine"}, - "count": {"count"}, - "tree": {"tree"}, - "vocabulary": {"vocabulary"}, - } - - def test_the_engine_call_scan_sees_any_receiver(tmp_path) -> None: import types @@ -1089,6 +1016,9 @@ def changed_module_bytes(path): # derive those modules from what a stage's transform, or a UK graph kernel's # own code, names, and follow each named definition into the module that # defines it, through re-exports and module attributes, to a fixed point. +# The walk is country-neutral (test_support.microcosm_build. +# derived_binding_rule); the tables below are the UK's, and the stage table, +# the engine root and the gate-battery phase walk are the UK's own. #: The packages the rule follows: every first-party package, each under the #: microcosm namespace (microcosm-build, -calibrate, -data, -diagnostics, @@ -1100,7 +1030,7 @@ def changed_module_bytes(path): #: weight-update receipt and content store; and microcosm.calibrate's solver, #: targets and region tier. It stops at the core modules every node runs #: through (``_CORE_MODULES``). -_HELPER_SCOPE = "microcosm." +_HELPER_SCOPE = FIRST_PARTY_SCOPE #: Where the rule stops. What a stage reads from these is its own #: declaration, which its node key already hashes as the stage contract: its #: operations, artifacts and resource pins (graph._stage_contract_sha256). @@ -1458,497 +1388,42 @@ def changed_module_bytes(path): "that artifact is absent", ), } -#: Packages that export names lazily, through a module ``__getattr__``, each -#: with the table, of name to the submodule defining it, that ``__getattr__`` -#: reads. The walk resolves such a name to the module that defines it and -#: follows it like any other (microcosm#931 ruling 8). microcosm.graph imports -#: its runtime modules, which need numpy, pandas and microcosm-frame, only on -#: first access. test_every_lazy_export_resolves_to_the_object_python_returns -#: checks the table. -_LAZY_EXPORT_TABLES: dict[str, str] = {"microcosm.graph": "_RUNTIME_EXPORTS"} -#: First-party modules that define a module ``__getattr__`` but export -#: nothing through it, each with why. -_NO_LAZY_EXPORTS: dict[str, str] = { - "microcosm.build.us_runtime.sources": ( - "its __getattr__ raises for every name: it only words the error for " - "the names it removed" - ), -} -#: Method and function names that change their receiver or an argument in -#: place. -_IN_PLACE_CALLS = frozenset( - { - "__delitem__", - "__setattr__", - "__setitem__", - "add", - "append", - "appendleft", - "at", - "clear", - "copyto", - "difference_update", - "discard", - "extend", - "extendleft", - "fill", - "fill_diagonal", - "insert", - "intersection_update", - "itemset", - "place", - "pop", - "popitem", - "put", - "putmask", - "remove", - "resize", - "reverse", - "rotate", - "setdefault", - "setflags", - "shuffle", - "sort", - "symmetric_difference_update", - "update", - } -) - - -class _ModuleDefinitions(NamedTuple): - """What one module imports, anywhere in it, and defines at its top level.""" - - imports: dict[str, tuple[str, str]] - modules: dict[str, str] - stars: tuple[str, ...] - definitions: dict[str, object] - - -@functools.cache -def _is_module(name: str) -> bool: - """Whether a first-party dotted name is a module. The rule never follows - another package, so it never imports one to find out.""" - - import importlib.util - - if not name.startswith("microcosm."): - return False - parent = name.rpartition(".")[0] - try: - if parent: - spec = importlib.util.find_spec(parent) - if spec is None or spec.submodule_search_locations is None: - return False - return importlib.util.find_spec(name) is not None - except (ImportError, ValueError): - return False - - -@functools.cache -def _module_definitions(name: str, scope: str) -> _ModuleDefinitions | None: - import ast - import importlib.util - - if not name.startswith(scope): - return None - try: - spec = importlib.util.find_spec(name) - except (ImportError, ValueError): - return None - if spec is None or not (spec.origin or "").endswith(".py"): - return None - package = name if spec.submodule_search_locations else name.rpartition(".")[0] - tree = ast.parse(Path(spec.origin).read_text(encoding="utf-8")) - imports: dict[str, tuple[str, str]] = {} - modules: dict[str, str] = {} - stars: list[str] = [] - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - if node.level: - base = package.rsplit(".", node.level - 1)[0] - origin = ".".join(part for part in (base, node.module) if part) - else: - origin = node.module or "" - for alias in node.names: - if alias.name == "*": - stars.append(origin) - elif _is_module(f"{origin}.{alias.name}"): - modules[alias.asname or alias.name] = f"{origin}.{alias.name}" - else: - imports[alias.asname or alias.name] = (origin, alias.name) - elif isinstance(node, ast.Import): - for alias in node.names: - if alias.asname: - modules[alias.asname] = alias.name - else: - head = alias.name.partition(".")[0] - modules[head] = head - definitions: dict[str, object] = {} - for node in tree.body: - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): - definitions[node.name] = node - elif isinstance(node, ast.TypeAlias): - definitions[node.name.id] = node - elif isinstance(node, ast.Assign | ast.AnnAssign): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - for target in targets: - for sub in ast.walk(target): - if isinstance(sub, ast.Name): - definitions[sub.id] = node - return _ModuleDefinitions(imports, modules, tuple(stars), definitions) - - -@functools.cache -def _lazy_exports(module: str) -> Mapping[str, str]: - """Each name ``module`` exports lazily, to the module defining it, read - from the table its ``__getattr__`` reads (``_LAZY_EXPORT_TABLES``).""" - - import importlib - - table = _LAZY_EXPORT_TABLES.get(module) - if table is None: - return MappingProxyType({}) - exports = getattr(importlib.import_module(module), table) - return MappingProxyType( - {name: f"{module}.{submodule}" for name, submodule in exports.items()} +#: The rule over the UK's tables (test_support.microcosm_build. +#: derived_binding_rule, whose controls are in +#: tests/engine_free/shared/test_derived_binding_rule.py). The lazy exports it +#: follows are the first-party packages' (``LAZY_EXPORT_TABLES`` there). +_RULE = BindingRule( + RuleTables( + core_modules=_CORE_MODULES, + refusal_only=_REFUSAL_ONLY_HELPERS, + stops=_KERNEL_STOPS, + declaration_modules=_DECLARATION_MODULES, ) - - -def _resolve(module: str, name: str, scope: str) -> tuple[str, str] | None: - """The top-level definition ``name`` refers to in ``module``, through - re-exports and lazy exports (``_LAZY_EXPORT_TABLES``); None for a module - or anything outside ``scope``.""" - - pending, seen = [(module, name)], set() - while pending: - item = pending.pop() - if item in seen: - continue - seen.add(item) - module, name = item - facts = _module_definitions(module, scope) - if facts is None: - continue - if name in facts.definitions: - return item - if name in facts.imports: - pending.append(facts.imports[name]) - continue - # As in Python, the module's __getattr__ supplies a name only where - # none of its own attributes, its star imports' included, is one. - lazy = _lazy_exports(module).get(name) - if lazy is not None: - pending.append((lazy, name)) - pending.extend((star, name) for star in facts.stars) - return None - - -def _named(module: str, node, scope: str) -> set[tuple[str, str]]: - """The top-level definitions ``node`` names, annotations included: a bare - name, or a name read by attribute on a module, each resolved through - re-exports.""" - - import ast - - facts = _module_definitions(module, scope) - found = set() - for sub in ast.walk(node): - if isinstance(sub, ast.Name): - hit = _resolve(module, sub.id, scope) - elif isinstance(sub, ast.Attribute): - chain = [] - value = sub - while isinstance(value, ast.Attribute): - chain.insert(0, value.attr) - value = value.value - target = ( - facts.modules.get(value.id) if isinstance(value, ast.Name) else None - ) - while target and chain and _is_module(f"{target}.{chain[0]}"): - target = f"{target}.{chain.pop(0)}" - hit = _resolve(target, chain[0], scope) if target and chain else None - else: - continue - if hit is not None: - found.add(hit) - return found - - -def _writes_nothing(node) -> bool: - """Whether a function can only refuse: it returns nothing but None, or its - sole parameter unrebound, and writes nothing: no store through a - subscript or attribute, no augmented assignment, no ``out=`` or - ``inplace=`` argument, and no call by a name that changes its receiver or - an argument in place.""" - - import ast - - arguments = node.args - parameters = { - argument.arg - for argument in ( - *arguments.posonlyargs, - *arguments.args, - *arguments.kwonlyargs, - *(a for a in (arguments.vararg, arguments.kwarg) if a is not None), - ) - } - rebound = { - sub.id - for sub in ast.walk(node) - if isinstance(sub, ast.Name) and isinstance(sub.ctx, ast.Store | ast.Del) - } - returnable = parameters - rebound if len(parameters) == 1 else set() - for sub in ast.walk(node): - if isinstance(sub, ast.Return) and sub.value is not None: - value = sub.value - if not ( - (isinstance(value, ast.Constant) and value.value is None) - or (isinstance(value, ast.Name) and value.id in returnable) - ): - return False - elif isinstance( - sub, ast.AugAssign | ast.Yield | ast.YieldFrom | ast.Global | ast.Nonlocal - ): - return False - elif isinstance(sub, ast.Assign | ast.AnnAssign | ast.Delete): - targets = ( - sub.targets - if isinstance(sub, ast.Assign | ast.Delete) - else [sub.target] - ) - if any( - isinstance(part, ast.Subscript | ast.Attribute) - for target in targets - for part in ast.walk(target) - ): - return False - elif isinstance(sub, ast.keyword) and sub.arg in {"inplace", "out"}: - return False - elif isinstance(sub, ast.Call): - function = sub.func - if isinstance(function, ast.Attribute) and function.attr in _IN_PLACE_CALLS: - return False - if isinstance(function, ast.Name) and function.id in { - *_IN_PLACE_CALLS, - "delattr", - "eval", - "exec", - "setattr", - }: - return False - return True - - -def _string_expression(node) -> bool: - """A string literal, an f-string of names or of one call on names, or a - concatenation of them.""" - - import ast - - if isinstance(node, ast.Constant): - return isinstance(node.value, str) - if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): - return _string_expression(node.left) and _string_expression(node.right) - if not isinstance(node, ast.JoinedStr): - return False - for part in node.values: - value = part.value if isinstance(part, ast.FormattedValue) else part - if isinstance(value, ast.Call): - if not isinstance(value.func, ast.Name) or value.keywords: - return False - value = value.args[0] if len(value.args) == 1 else None - if not isinstance(value, ast.Name | ast.Constant): - return False - return True - - -_HELPER_KINDS: dict[tuple[str, str, str], str | None] = {} - - -def _helper_kind(module: str, name: str, scope: str) -> str: - """How a top-level definition can bear on a value. - - ``value`` unless it is a listed refusal-only ``check`` - (``_REFUSAL_ONLY_HELPERS``) or provably one of: a ``name`` (one string: a - literal, an f-string or concatenation of names, or another name of these - kinds); a ``column name`` (a function returning one such string built - from its arguments); a ``record`` (a class of undefaulted fields whose - types are of these kinds, no methods). A collection of strings is a - value, since it selects rows, columns or codes, as is a number. A cycle - reads as ``value``. - """ - - if (module, name) in _REFUSAL_ONLY_HELPERS: - return "check" - key = (module, name, scope) - if key in _HELPER_KINDS: - return _HELPER_KINDS[key] or "value" - _HELPER_KINDS[key] = None - _HELPER_KINDS[key] = _classify_helper(module, name, scope) - return _HELPER_KINDS[key] - - -def _names_only_harmless(module: str, node, scope: str) -> bool: - return all( - found[0].startswith(_DECLARATION_MODULES) - or _helper_kind(*found, scope) != "value" - for found in _named(module, node, scope) - ) - - -def _classify_helper(module: str, name: str, scope: str) -> str: - import ast - - node = _module_definitions(module, scope).definitions[name] - # A type alias statement (``type X = ...``) reads as an assignment of its - # value. - if isinstance(node, ast.Assign | ast.AnnAssign | ast.TypeAlias): - if node.value is None or ( - ( - isinstance(node.value, ast.Name | ast.Attribute) - or _string_expression(node.value) - ) - and _names_only_harmless(module, node.value, scope) - ): - return "name" - return "value" - if isinstance(node, ast.ClassDef): - fields = [item for item in node.body if isinstance(item, ast.AnnAssign)] - fields_only = all( - (isinstance(item, ast.Expr) and isinstance(item.value, ast.Constant)) - or isinstance(item, ast.Pass) - or (isinstance(item, ast.AnnAssign) and item.value is None) - for item in node.body - ) - if fields_only and all( - _names_only_harmless(module, item, scope) - for item in ( - *node.bases, - *node.keywords, - *node.decorator_list, - *(field.annotation for field in fields), - ) - ): - return "record" - return "value" - body = list(node.body) - if ( - body - and isinstance(body[0], ast.Expr) - and isinstance(body[0].value, ast.Constant) - ): - body = body[1:] - if ( - len(body) == 1 - and isinstance(body[0], ast.Return) - and _string_expression(body[0].value) - and _names_only_harmless(module, body[0].value, scope) - ): - return "column name" - return "value" - - -class _Walk(NamedTuple): - """What a root can compute through (``_helper_walk``).""" - - reach: dict[str, frozenset[str]] - stops: frozenset[tuple[str, str]] - core: frozenset[str] - - -def _helper_walk( - root: tuple[str, str], scope: str = _HELPER_SCOPE, *, follow_core: bool = False -) -> _Walk: - """What ``root`` can compute through, where the rule stops, and the core - modules it meets. - - ``reach`` maps every module holding a definition that can change what - ``root`` computes to those definitions: ``root`` itself, and each - ``value`` definition it names, and they name, to a fixed point. ``stops`` - is every ``name``, ``column name`` or ``record`` those definitions name, - which the rule does not follow. ``core`` is every listed core module - (``_CORE_MODULES``) whose definitions they name, which the rule neither - follows nor returns in ``reach``, unless ``follow_core``. The declaration - modules and the listed refusal-only checks are neither followed nor - returned.""" - - module, name = root - node = _module_definitions(module, scope).definitions[name] - walk, _, _ = _walk( - {root}, - [(found, root) for found in _named(module, node, scope)], - scope, - follow_core=follow_core, - ) - return walk - - -def _walk( - roots: set[tuple[str, str]], - seeds: list[tuple[tuple[str, str], object]], - scope: str, - *, - follow_core: bool = False, - halts: Mapping[tuple[str, str], str] = MappingProxyType({}), -) -> tuple[_Walk, dict[tuple[str, str], frozenset]]: - """The walk behind ``_helper_walk`` and ``_kernel_walk``: ``roots`` are - reached as given, and each ``(definition, namer)`` seed is followed as - ``_helper_walk`` follows what its root names. A definition in ``halts`` is - neither followed nor returned; the second result maps each one met to - what named it, and the third holds every core module's definition met.""" - - reached: dict[str, set[str]] = {} - for module, name in roots: - reached.setdefault(module, set()).add(name) - stops: set[tuple[str, str]] = set() - core: set[str] = set() - core_names: set[tuple[str, str]] = set() - met: dict[tuple[str, str], set] = {} - pending, seen = list(seeds), set(roots) - while pending: - item, namer = pending.pop() - if item in halts: - met.setdefault(item, set()).add(namer) - continue - if item in seen: - continue - seen.add(item) - module, name = item - if module.startswith(_DECLARATION_MODULES): - continue - if module in _CORE_MODULES: - core.add(module) - core_names.add(item) - if not follow_core: - continue - kind = _helper_kind(module, name, scope) - if kind != "value": - if kind != "check": - stops.add(item) - continue - reached.setdefault(module, set()).add(name) - node = _module_definitions(module, scope).definitions[name] - pending.extend((found, item) for found in _named(module, node, scope)) - walk = _Walk( - {module: frozenset(names) for module, names in reached.items()}, - frozenset(stops), - frozenset(core), - ) - return ( - walk, - {item: frozenset(namers) for item, namers in met.items()}, - frozenset(core_names), - ) - - -def _helper_reach( - root: tuple[str, str], scope: str = _HELPER_SCOPE -) -> dict[str, frozenset[str]]: - """Every module holding a definition that can change what ``root`` - computes, with those definitions (``_helper_walk``).""" - - return _helper_walk(root, scope).reach +) +# The shared machinery and the rule's methods, under the names the UK code +# below uses. +_Walk = Walk +_KernelWalk = KernelWalk +_KeyedWalk = KeyedWalk +_module_definitions = module_definitions +_class_named = class_named +_merged = merged +_hashed_by = hashed_by +_hashed_modules = hashed_modules +_resolve = _RULE.resolve +_named = _RULE.named +_helper_kind = _RULE.helper_kind +_walk = _RULE.walk +_helper_walk = _RULE.helper_walk +_helper_reach = _RULE.helper_reach +_kernel_classes = _RULE.kernel_classes +_kernel_body = _RULE.kernel_body +_kernel_walk = _RULE.kernel_walk +_engine_methods = _RULE.engine_methods +_engine_roots = _RULE.engine_roots +_engine_seeds = _RULE.engine_seeds +_engine_methods_beyond = _RULE.engine_methods_beyond +_engine_reach = _RULE.engine_reach #: The class of the engine the builds inject into the stages that compute @@ -1958,155 +1433,10 @@ def _helper_reach( _ENGINE_CLASS = ("microcosm.frame.adapters.policyengine_uk", "PolicyEngineUKEngine") -def _class_named(ref: tuple[str, str]) -> type: - import importlib - - module, name = ref - return getattr(importlib.import_module(module), name) - - def _engine_class() -> type: return _class_named(_ENGINE_CLASS) -def _class_attributes(cls: type) -> dict[str, list[tuple[type, object]]]: - """Each attribute ``cls`` defines, with the class and statement that - define it, as ``_kernel_body`` chooses them by method resolution order.""" - - body: dict[str, list[tuple[type, object]]] = {} - for owner, statement, name in _kernel_body(cls): - body.setdefault(name, []).append((owner, statement)) - return body - - -def _engine_methods(engine_class: type, walk: _Walk) -> frozenset[str]: - """The attributes of ``engine_class`` that the code ``walk`` reaches - names: read by attribute, on any receiver, or through ``getattr`` or - ``hasattr`` with a constant name. It matches by name alone, as - ``_calls_the_engine`` matches its calls, so it sees the engine under any - name a stage holds it by, and can only over-count.""" - - import ast - - found: set[object] = set() - for module, names in walk.reach.items(): - definitions = _module_definitions(module, _HELPER_SCOPE).definitions - for name in names: - for sub in ast.walk(definitions[name]): - if isinstance(sub, ast.Attribute): - found.add(sub.attr) - elif ( - isinstance(sub, ast.Call) - and isinstance(sub.func, ast.Name) - and sub.func.id in {"getattr", "hasattr"} - and len(sub.args) > 1 - and isinstance(sub.args[1], ast.Constant) - ): - found.add(sub.args[1].value) - return frozenset(found & set(_class_attributes(engine_class))) - - -def _engine_roots(engine_class: type) -> frozenset[tuple[str, str]]: - """The classes an engine of ``engine_class`` comes from, the roots of its - walk. Their definitions name every method, so the engine methods a walk - names are read outside them (``_engine_reach``).""" - - return frozenset( - (cls.__module__, cls.__name__) for cls in _kernel_classes(engine_class) - ) - - -def _engine_seeds( - engine_class: type, methods: frozenset[str], scope: str = _HELPER_SCOPE -) -> list[tuple[tuple[str, str], tuple[str, str]]]: - """What ``methods`` of ``engine_class``, the constructor the builds call - and every method they call on ``self`` name, each with the method naming - it: the seeds of the engine walk (``_engine_walk``).""" - - import ast - - body = _class_attributes(engine_class) - pending, chosen = sorted({"__init__", *methods} & set(body)), set() - while pending: - name = pending.pop() - if name in chosen: - continue - chosen.add(name) - for _, statement in body[name]: - pending.extend( - sub.attr - for sub in ast.walk(statement) - if isinstance(sub, ast.Attribute) - and isinstance(sub.value, ast.Name) - and sub.value.id in {"self", "cls"} - and sub.attr in body - ) - return [ - (found, (owner.__module__, f"{owner.__name__}.{name}")) - for name in sorted(chosen) - for owner, statement in body[name] - for found in _named(owner.__module__, statement, scope) - ] - - -def _engine_walk( - engine_class: type, methods: frozenset[str], scope: str = _HELPER_SCOPE -) -> _Walk: - """What an engine of ``engine_class`` computes through for a stage that - calls ``methods`` on it: the stage rule's walk, at ``scope`` with the - same tables, from those methods, the constructor the builds call, and - every method they call on ``self``, to a fixed point.""" - - walk, _, _ = _walk( - set(_engine_roots(engine_class)), - _engine_seeds(engine_class, methods, scope), - scope, - ) - return walk - - -def _engine_methods_beyond(engine_class: type, walk: _Walk) -> frozenset[str]: - """The methods of ``engine_class`` that ``walk`` names outside the - engine's own classes (``_engine_methods``, ``_engine_roots``).""" - - roots = _engine_roots(engine_class) - return _engine_methods( - engine_class, - _Walk( - { - module: frozenset(name for name in names if (module, name) not in roots) - for module, names in walk.reach.items() - }, - walk.stops, - walk.core, - ), - ) - - -def _engine_reach(engine_class: type, walk: _Walk, scope: str = _HELPER_SCOPE) -> _Walk: - """What an engine of ``engine_class`` computes through for code whose - walk is ``walk``: the engine walk (``_engine_walk``) from the methods - ``walk`` names, and from those the engine walk's own reach names on any - receiver outside the engine's classes, to a fixed point. So a helper the - engine hands itself to, which calls another of its methods on that - argument, is followed into that method too.""" - - methods = _engine_methods(engine_class, walk) - while True: - reached = _engine_walk(engine_class, methods, scope) - more = methods | _engine_methods_beyond(engine_class, reached) - if more == methods: - return reached - methods = more - - -def _merged(first: _Walk, second: _Walk) -> _Walk: - reach = dict(first.reach) - for module, names in second.reach.items(): - reach[module] = reach.get(module, frozenset()) | names - return _Walk(reach, first.stops | second.stops, first.core | second.core) - - def _stage_transforms() -> dict[str, tuple[str, str]]: """Each spine stage's transform, as graph_kernels builds it for the hermetic registry and the legacy plan: ``(module, name)``.""" @@ -2347,9 +1677,8 @@ def test_the_core_modules_are_on_every_nodes_path() -> None: every spine stage kernel declares SeedSource.NONE.""" from microcosm.graph import SeedSource - executor = _helper_walk(("microcosm.graph.executor", "run_graph"), follow_core=True) - on_path = set(executor.reach) | set(executor.core) - assert set(_CORE_MODULES) <= on_path, sorted(set(_CORE_MODULES) - on_path) + off_path = _RULE.core_modules_off_run_graph_path() + assert not off_path, off_path frame = { module for module in _CORE_MODULES if module.startswith("microcosm.frame.") } @@ -2375,417 +1704,8 @@ def test_the_core_modules_are_on_every_nodes_path() -> None: def test_the_refusal_only_helpers_return_and_write_nothing() -> None: - import ast - - for module, name in _REFUSAL_ONLY_HELPERS: - owner, _, method = name.rpartition(".") - node = _module_definitions(module, _HELPER_SCOPE).definitions[owner or name] - if owner: - (node,) = [ - item - for item in node.body - if isinstance(item, ast.FunctionDef) and item.name == method - ] - assert _writes_nothing(node), (module, name) - - -def test_the_refusal_check_sees_each_way_a_function_writes() -> None: - """Control for ``_writes_nothing``: a function that only refuses, or - returns its sole argument, passes; one that writes through any of these - shapes, or returns one of several arguments, does not.""" - import ast - - refusing = { - "raise": "def f(frame):\n if frame is None:\n raise ValueError('x')\n", - "identity": ( - "def f(entity):\n" - " if entity not in {'person'}:\n" - " raise ValueError('x')\n" - " return entity\n" - ), - } - writing = { - "copyto": "def f(values, mask):\n np.copyto(values, 0.0, where=mask)\n", - "out": "def f(values):\n np.clip(values, 0, 1, out=values)\n", - "add_at": "def f(w, roots, copies):\n np.add.at(w, roots, w[copies])\n", - "shuffle": "def f(rng, values):\n rng.shuffle(values)\n", - "reverse": "def f(items):\n items.reverse()\n", - "augmented": "def f(values):\n values *= 2.0\n", - "subscript": "def f(values):\n values[0] = 1.0\n", - "inplace": "def f(table):\n table.drop(columns=['a'], inplace=True)\n", - "pick": "def f(left, right):\n return left\n", - "computed": "def f(values):\n return values + 1\n", - } - for label, source in {**refusing, **writing}.items(): - function = ast.parse(source).body[0] - assert _writes_nothing(function) is (label in refusing), label - - -def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( - tmp_path, monkeypatch -) -> None: - """Control: a helper called by name, through a re-export, a lazy export, - by attribute on its module, through an alias or a computed constant, or - held only in an annotation (a record's field type included) is reached, - as are a numeric constant, a collection of strings and a function that - writes in place or only refuses but is not listed; a listed check, a - single string, a column name, a record of plain fields, a listed core - module, what only a core module names, and a helper the root never names - are not. The walk returns the names it stopped at and the core modules it - met, and follows a core module only when asked to.""" - import uuid - - scratch = f"helper_rule_{uuid.uuid4().hex[:8]}" - package = tmp_path / "microcosm" / scratch - package.mkdir(parents=True) - (package / "__init__.py").write_text("") - # A package exporting its runtime module's names lazily, as - # microcosm.graph does. - (package / "lazily").mkdir() - (package / "lazily" / "__init__.py").write_text( - "from importlib import import_module\n\n" - "_EXPORTS = {'deferred': 'runtime', 'idle': 'runtime'}\n\n" - "def __getattr__(name):\n" - " if name not in _EXPORTS:\n" - " raise AttributeError(name)\n" - " return getattr(import_module(f'{__name__}.{_EXPORTS[name]}'), name)\n" - ) - (package / "lazily" / "runtime.py").write_text( - "def deferred(x):\n return x * 6\n\ndef idle(x):\n return x * 7\n" - ) - monkeypatch.syspath_prepend(str(tmp_path)) - sources = { - "direct": "def draw(x):\n return x + 1\n", - "origin": "def total(x):\n return sum(x)\n", - "reexport": "from .origin import total\n", - "attribute": "def share(x):\n return x / 2\n", - "alias": "def _scale(x):\n return x * 3\n\nscale = _scale\n", - "computed": "def _build():\n return {'a': 1}\n\nTABLE = _build()\n", - "collaborator": ( - "class Contract:\n def rate(self, key):\n return 0.5\n" - ), - "fielded": ( - "from typing import NamedTuple\n\n" - "class Rate:\n def value(self):\n return 0.1\n\n" - "class Inputs(NamedTuple):\n rate: Rate\n" - ), - "numeric": "FLOOR = 10\n", - "codes": ( - "CODES = frozenset({'A', 'B'})\n" - "OWNER = {'OWNED': 'owner', 'RENTED': 'renter'}\n" - ), - "inplace": ( - "import numpy as np\n\n" - "def fold(weights, roots, copies):\n" - " np.add.at(weights, roots, weights[copies])\n" - ), - "guard": ( - "def guard(frame):\n" - " if frame is None:\n" - " raise ValueError('no frame')\n" - ), - "harmless": ( - "COLUMN = 'household_weight'\n\n" - "def check(frame):\n" - " if frame is None:\n" - " raise ValueError('no frame')\n\n" - "def column(entity):\n" - " return f'{entity}_support_channel'\n\n" - "class Record:\n" - " value: float\n" - ), - "unused": "def never(x):\n return x - 1\n", - "behind": "def kept(x):\n return x * 4\n", - "core": ( - "from .behind import kept\n\n" - "class Container:\n" - " def grow(self, x):\n" - " return kept(x)\n" - ), - "stage": ( - "from . import attribute\n" - "from .alias import scale\n" - "from .codes import CODES, OWNER\n" - "from .collaborator import Contract\n" - "from .computed import TABLE\n" - "from .core import Container\n" - "from .direct import draw\n" - "from .fielded import Inputs\n" - "from .guard import guard\n" - "from .harmless import COLUMN, Record, check, column\n" - "from .inplace import fold\n" - "from .lazily import deferred\n" - "from .numeric import FLOOR\n" - "from .reexport import total\n" - "from .unused import never\n\n" - "class Transform:\n" - " contract: Contract\n" - " inputs: Inputs\n\n" - " def __call__(self, frame):\n" - " check(frame)\n" - " guard(frame)\n" - " fold(frame, [0], [1])\n" - " record = Record(value=draw(1) + total([1]) + deferred(1))\n" - " rate = self.contract.rate(column('household'))\n" - " rate += self.inputs.rate.value()\n" - " owners = [OWNER[code] for code in CODES]\n" - " rate += Container().grow(rate)\n" - " return attribute.share(scale(rate)) + TABLE['a'] + FLOOR, (\n" - " record, COLUMN, owners\n" - " )\n\n" - "def helper():\n" - " return never(1)\n" - ), - } - for module, source in sources.items(): - (package / f"{module}.py").write_text(source) - prefix = f"microcosm.{scratch}." - monkeypatch.setitem( - _REFUSAL_ONLY_HELPERS, (f"{prefix}harmless", "check"), "control" - ) - monkeypatch.setitem(_CORE_MODULES, f"{prefix}core", "control") - monkeypatch.setitem(_LAZY_EXPORT_TABLES, f"{prefix}lazily", "_EXPORTS") - reach, stops, core = _helper_walk((f"{prefix}stage", "Transform"), scope=prefix) - assert {module.removeprefix(prefix): names for module, names in reach.items()} == { - "stage": {"Transform"}, - "direct": {"draw"}, - "origin": {"total"}, - "lazily.runtime": {"deferred"}, - "attribute": {"share"}, - "alias": {"scale", "_scale"}, - "computed": {"TABLE", "_build"}, - "collaborator": {"Contract"}, - "fielded": {"Inputs", "Rate"}, - "numeric": {"FLOOR"}, - "codes": {"CODES", "OWNER"}, - "inplace": {"fold"}, - "guard": {"guard"}, - } - assert {(module.removeprefix(prefix), name) for module, name in stops} == { - ("harmless", "COLUMN"), - ("harmless", "Record"), - ("harmless", "column"), - } - assert core == {f"{prefix}core"} - followed = _helper_walk( - (f"{prefix}stage", "Transform"), scope=prefix, follow_core=True - ) - assert set(followed.reach) - set(reach) == {f"{prefix}core", f"{prefix}behind"} - assert followed.core == core - - -class _KernelWalk(NamedTuple): - """What a kernel's own code can compute through (``_kernel_walk``).""" - - reach: dict[str, frozenset[str]] - stops: frozenset[tuple[str, str]] - core: frozenset[str] - met: dict[tuple[str, str], frozenset] - core_names: frozenset[tuple[str, str]] - - -def _kernel_classes(kernel_class: type) -> list[type]: - """The first-party classes a kernel's code comes from, in its method - resolution order, without the core modules' (``KernelBase``).""" - - return [ - cls - for cls in kernel_class.__mro__ - if cls.__module__.startswith(_HELPER_SCOPE) - and cls.__module__ not in _CORE_MODULES - ] - - -def _calls_super(node) -> bool: - import ast - - return any( - isinstance(sub, ast.Call) - and isinstance(sub.func, ast.Name) - and sub.func.id == "super" - for sub in ast.walk(node) - ) - - -def _statement_names(statement) -> list[str]: - import ast - - if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef): - return [statement.name] - if isinstance(statement, ast.Assign | ast.AnnAssign): - targets = ( - statement.targets - if isinstance(statement, ast.Assign) - else [statement.target] - ) - return [ - sub.id - for target in targets - for sub in ast.walk(target) - if isinstance(sub, ast.Name) - ] - return [] - - -def _kernel_body(kernel_class: type) -> list[tuple[type, object, str]]: - """What a kernel of ``kernel_class`` runs: each method or class attribute - with its class and name, from the first class in the method resolution - order that defines it, as Python resolves it, and a base's definition too - where the one that wins calls ``super()``. ``implementation_hash`` is left - out, since it computes the key: an edit to what it calls that changes what - it returns changes the key. So is a method only the key computation calls - (``_key_only_methods``), such as the full gates' ``_country_resources``, - and so are the listed refusal-only methods.""" - - chosen: dict[str, object | None] = {} - body = [] - keyed = [] - for cls in _kernel_classes(kernel_class): - definitions = _module_definitions(cls.__module__, _HELPER_SCOPE).definitions - for statement in definitions[cls.__name__].body: - for name in _statement_names(statement): - if name in chosen: - previous = chosen[name] - if previous is not None and _calls_super(previous): - chosen[name] = statement - (keyed if name == "implementation_hash" else body).append( - (cls, statement, name) - ) - continue - if (cls.__module__, f"{cls.__name__}.{name}") in _REFUSAL_ONLY_HELPERS: - chosen[name] = None - continue - chosen[name] = statement - (keyed if name == "implementation_hash" else body).append( - (cls, statement, name) - ) - key_only = _key_only_methods(body, keyed) - return [item for item in body if item[2] not in key_only] - - -def _self_attributes(statement) -> set[str]: - """The attributes ``statement`` reads on ``self``, ``cls`` or - ``type(self)``.""" - import ast - - found = set() - for sub in ast.walk(statement): - if not isinstance(sub, ast.Attribute): - continue - value = sub.value - if (isinstance(value, ast.Name) and value.id in {"self", "cls"}) or ( - isinstance(value, ast.Call) - and isinstance(value.func, ast.Name) - and value.func.id == "type" - ): - found.add(sub.attr) - return found - - -def _key_only_methods(body, keyed) -> frozenset[str]: - """The methods of a kernel's ``body`` that only its key computation calls: - named on ``self``, ``cls`` or ``type(self)`` by its ``keyed`` statements - (``implementation_hash``) or by another such method, and by nothing else - the kernel runs. They compute the key as the hash does.""" - import ast - - methods = { - name - for _, statement, name in body - if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef) - } - reads: dict[str, set[str]] = {} - for _, statement, name in body: - reads.setdefault(name, set()).update(_self_attributes(statement)) - key = set().union(*(_self_attributes(statement) for _, statement, _ in keyed)) - key_only: set[str] = set() - while True: - found = { - method - for method in methods - key_only - if method in key.union(*(reads[item] for item in key_only)) - and not any( - method in names - for other, names in reads.items() - if other != method and other not in key_only - ) - } - if not found: - return frozenset(key_only) - key_only |= found - - -def _kernel_walk( - kernel_class: type, - scope: str = _HELPER_SCOPE, - *, - halts: Mapping[tuple[str, str], str] | None = None, -) -> _KernelWalk: - """What a kernel's own code can compute through: the rule followed from - what ``_kernel_body`` names, as from a stage's transform, stopping also at - ``halts`` (``_KERNEL_STOPS`` unless given). Its classes are in ``reach``; - ``met`` maps each listed stop it meets to what names it there (a - definition, or ``Class.attribute`` for the kernel's own code), and - ``core_names`` holds the core modules' definitions it names.""" - - classes = _kernel_classes(kernel_class) - seeds = [ - (found, (cls.__module__, f"{cls.__name__}.{name}")) - for cls, statement, name in _kernel_body(kernel_class) - for found in _named(cls.__module__, statement, scope) - ] - walk, met, core_names = _walk( - {(cls.__module__, cls.__name__) for cls in classes}, - seeds, - scope, - halts=_KERNEL_STOPS if halts is None else halts, - ) - return _KernelWalk(walk.reach, walk.stops, walk.core, met, core_names) - - -def _hashed_by(compute: Callable[[], object]) -> frozenset[str]: - """Every module ``compute()`` hashes, recorded at each ``source_hash`` it - reaches, in whichever module calls it: every loaded module's - ``source_hash`` name stands in for the recorder during one call, after a - first call has imported what ``compute`` imports.""" - - import inspect - import sys - from types import ModuleType - - from microcosm.graph import kernel as graph_kernel - - original = graph_kernel.source_hash - recorded: list[object] = [] - - def record(*objects, **dependencies): - recorded.extend(objects) - return original(*objects, **dependencies) - - compute() - patched = [ - module - for module in list(sys.modules.values()) - if isinstance(module, ModuleType) - and getattr(module, "__dict__", {}).get("source_hash") is original - ] - for module in patched: - module.source_hash = record - try: - compute() - finally: - for module in patched: - module.source_hash = original - return frozenset( - (item if isinstance(item, ModuleType) else inspect.getmodule(item)).__name__ - for item in recorded - ) - - -def _hashed_modules(kernel) -> frozenset[str]: - """Every module ``kernel.implementation_hash()`` hashes (``_hashed_by``).""" - - return _hashed_by(kernel.implementation_hash) + violations = _RULE.refusal_only_violations() + assert not violations, violations @functools.cache @@ -3147,14 +2067,6 @@ def _phase_code_modules(battery: _Battery, phase: str) -> frozenset[str]: return _hashed_by(getattr(code, "__wrapped__", code)) -class _KeyedWalk(NamedTuple): - """A walk the kernel rule holds a key to, and the modules that key - hashes.""" - - walk: _KernelWalk - key: frozenset[str] - - @functools.cache def _kernel_rule_walks() -> MappingProxyType: """What the kernel rule holds each UK graph kernel to, by ref and label: @@ -3218,34 +2130,20 @@ def test_the_kernel_rule_sees_every_kernel_the_uk_runtime_defines() -> None: and the shared atomic geography define, so a new one cannot escape it, and the gate batteries' and whole-registry kernels are among them, each listed once (microcosm#931).""" - import ast - import importlib - from microcosm.build import graph_atomic_geography from microcosm.build.uk_runtime import graph_kernels files = sorted(Path(graph_kernels.__file__).parent.glob("*.py")) files.append(Path(graph_atomic_geography.__file__)) - defined = set() - for path in files: - module = ( + defined = kernel_classes_defined( + ( graph_atomic_geography.__name__ if path.stem == "graph_atomic_geography" - else f"{graph_kernels.__package__}.{path.stem}" + else f"{graph_kernels.__package__}.{path.stem}", + path, ) - for node in ast.parse(path.read_text(encoding="utf-8")).body: - if not isinstance(node, ast.ClassDef): - continue - declares_ref = any( - "ref" in _statement_names(statement) for statement in node.body - ) or any( - isinstance(sub, ast.Attribute) - and sub.attr == "ref" - and isinstance(sub.ctx, ast.Store) - for sub in ast.walk(node) - ) - if declares_ref: - defined.add(getattr(importlib.import_module(module), node.name)) + for path in files + ) kernels = _uk_graph_kernels() assert {type(kernel) for kernel in kernels.values()} == defined listed = (set(_GATE_BATTERY_KERNELS), set(_WHOLE_REGISTRY_KERNELS)) @@ -3276,7 +2174,7 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: 6), and test_uk_target_compile_modules keeps only what it does not cover. A name microcosm.graph exports lazily, through its module ``__getattr__``, is followed to the runtime module that defines it like any other - (``_LAZY_EXPORT_TABLES``): the kernels' capability and context records are + (``LAZY_EXPORT_TABLES``): the kernels' capability and context records are listed core names, and the create kernel's ``load_source``, which it calls only where no root transform is bound, is a listed stop.""" import ast @@ -3291,21 +2189,12 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: ) kernels = _uk_graph_kernels() - unhashed: dict[str, list[str]] = {} - unlisted: set[tuple[str, str]] = set() - met: set[tuple[str, str]] = set() - for walks in _kernel_rule_walks().values(): - for label, (walk, key) in walks.items(): - missing = sorted(set(walk.reach) - key) - if missing: - unhashed[label] = missing - unlisted.update(item for item in walk.stops if item[0] not in key) - met |= set(walk.met) - assert not unhashed, unhashed - assert unlisted == set(_KERNEL_UNBOUND_NAMES), sorted( - unlisted ^ set(_KERNEL_UNBOUND_NAMES) + found = findings(_kernel_rule_walks()) + assert not found.unhashed, found.unhashed + assert found.unlisted == set(_KERNEL_UNBOUND_NAMES), sorted( + found.unlisted ^ set(_KERNEL_UNBOUND_NAMES) ) - assert met == set(_KERNEL_STOPS), sorted(met ^ set(_KERNEL_STOPS)) + assert found.met == set(_KERNEL_STOPS), sorted(found.met ^ set(_KERNEL_STOPS)) # The clone index columns' reason for the measures. The dense measures' # population is the expand's: append_uk_target_nodes defaults it, and the # full build leaves the default. The national problem resolves its @@ -3836,71 +2725,12 @@ def test_the_core_definitions_the_kernels_name_are_listed() -> None: the UK graph kernels' own code names are exactly ``_KERNEL_CORE_NAMES``, and the executor stores every node's owned cells through the dtype tokens' map, as that entry's reason says.""" - named = { - item - for walks in _kernel_rule_walks().values() - for keyed in walks.values() - for item in keyed.walk.core_names - if not item[0].startswith("microcosm.frame.") - } + named = findings(_kernel_rule_walks()).core_names assert named == set(_KERNEL_CORE_NAMES), sorted(named ^ set(_KERNEL_CORE_NAMES)) patching = _helper_walk(("microcosm.graph.population", "patch"), follow_core=True) assert "dtype_for_token" in patching.reach["microcosm.graph.population"] -def test_every_lazy_export_resolves_to_the_object_python_returns() -> None: - """The walk follows a name a package exports lazily, through its module - ``__getattr__``, to the module that defines it, as Python imports it - (microcosm#931 ruling 8). Every name microcosm.graph exports that way - resolves to a definition holding the very object the package returns, - the table the walk reads is the one the package's ``__getattr__`` reads, - and a name the package imports eagerly resolves as before. Every - first-party module that defines a module ``__getattr__`` is listed, as a - lazy exporter or as one that exports nothing through it, so a new one - cannot hide names from the walk.""" - import ast - import importlib - - import microcosm - from microcosm import graph - - for module, table in _LAZY_EXPORT_TABLES.items(): - getter = _module_definitions(module, _HELPER_SCOPE).definitions["__getattr__"] - assert table in { - sub.id for sub in ast.walk(getter) if isinstance(sub, ast.Name) - }, module - exports = getattr(graph, _LAZY_EXPORT_TABLES[graph.__name__]) - assert {"load_source", "KernelContext", "keyed_uniform"} <= set(exports) - for name in exports: - found = _resolve(graph.__name__, name, _HELPER_SCOPE) - assert found is not None, name - module, defined = found - assert getattr(importlib.import_module(module), defined) is getattr( - graph, name - ), name - assert _resolve(graph.__name__, "compile_graph", _HELPER_SCOPE) == ( - "microcosm.graph.decl", - "compile_graph", - ) - getters = set() - for root in map(Path, microcosm.__path__): - for path in sorted(root.rglob("*.py")): - text = path.read_text(encoding="utf-8") - if "def __getattr__" not in text or not any( - isinstance(node, ast.FunctionDef) and node.name == "__getattr__" - for node in ast.parse(text).body - ): - continue - parts = path.relative_to(root.parent).with_suffix("").parts - getters.add(".".join(parts[:-1] if parts[-1] == "__init__" else parts)) - listed = set(_LAZY_EXPORT_TABLES) | set(_NO_LAZY_EXPORTS) - assert getters == listed, sorted(getters ^ listed) - # The modules that export nothing that way only raise from __getattr__. - for module in _NO_LAZY_EXPORTS: - getter = _module_definitions(module, _HELPER_SCOPE).definitions["__getattr__"] - assert not any(isinstance(sub, ast.Return) for sub in ast.walk(getter)) - - def test_no_spine_or_national_line_kernel_hashes_the_source_codecs() -> None: """The create kernel's ``load_source`` stop keeps the spine's keys off microcosm.graph.codecs (microcosm#931 ruling 8): no spine kernel hashes @@ -3962,104 +2792,6 @@ def test_the_frame_totals_the_uk_corrections_target_are_numpys_sums() -> None: assert differing -def test_the_kernel_rule_follows_what_a_kernel_runs(tmp_path, monkeypatch) -> None: - """Control: the kernel rule follows what a kernel's methods name, - through its bases by method resolution order (a base method it inherits, - and one it overrides only where it calls ``super()``), and leaves out its - implementation hash, a method only the hash calls, a listed refusal-only - method, a base method it overrides without ``super()`` and what lies - behind a listed kernel stop, which it reports with what named it. A - method the hash and the run both call stays. The recorder sees each - module the hash passes to ``source_hash``, in whichever module calls - it.""" - import importlib - import uuid - - scratch = f"kernel_rule_{uuid.uuid4().hex[:8]}" - package = tmp_path / "microcosm" / scratch - package.mkdir(parents=True) - (package / "__init__.py").write_text("") - monkeypatch.syspath_prepend(str(tmp_path)) - sources = { - "computed": "def computed(x):\n return x + 1\n", - "inherited": "def base_only(x):\n return x * 2\n", - "assisted": "def kept(x):\n return x * 3\n", - "overridden": "def replaced(x):\n return x - 2\n", - "refused": "def check(x):\n return x < 0\n", - "keyonly": "def salt():\n return 'salt'\n", - "behind": "def beyond(x):\n return x * 5\n", - "resourced": "def fingerprint(x):\n return x * 7\n", - "shared": "def both(x):\n return x * 11\n", - "table": "from .behind import beyond\n\nTABLE = {'a': beyond}\n", - "keyed": ( - "from microcosm.graph import source_hash\n\n" - "from . import computed, keyonly\n\n" - "def key():\n return source_hash(computed, keyonly)\n" - ), - "base": ( - "from .assisted import kept\n" - "from .inherited import base_only\n" - "from .overridden import replaced\n\n" - "class Base:\n" - " def run(self, x):\n" - " return base_only(x)\n\n" - " def prepare(self, x):\n" - " return replaced(x)\n\n" - " def assist(self, x):\n" - " return kept(x)\n" - ), - "kernel": ( - "from .base import Base\n" - "from .computed import computed\n" - "from .keyed import key\n" - "from .keyonly import salt\n" - "from .refused import check\n" - "from .resourced import fingerprint\n" - "from .shared import both\n" - "from .table import TABLE\n\n" - "class Kernel(Base):\n" - " def implementation_hash(self):\n" - " return salt() + key() + self._resources(1) + self._both(1)\n\n" - " def _resources(self, x):\n" - " return str(fingerprint(x))\n\n" - " def _both(self, x):\n" - " return str(both(x))\n\n" - " def _admit(self, x):\n" - " if check(x):\n" - " raise ValueError('refused')\n\n" - " def prepare(self, x):\n" - " return x\n\n" - " def run(self, x):\n" - " self._admit(x)\n" - " self._both(x)\n" - " return computed(x) + super().run(x) + TABLE['a'](x)\n" - ), - } - for module, source in sources.items(): - (package / f"{module}.py").write_text(source) - prefix = f"microcosm.{scratch}." - monkeypatch.setitem( - _REFUSAL_ONLY_HELPERS, (f"{prefix}kernel", "Kernel._admit"), "control" - ) - monkeypatch.setitem(_KERNEL_STOPS, (f"{prefix}table", "TABLE"), "control") - kernel_class = importlib.import_module(f"{prefix}kernel").Kernel - walk = _kernel_walk(kernel_class, scope=prefix) - assert { - module.removeprefix(prefix): names for module, names in walk.reach.items() - } == { - "kernel": {"Kernel"}, - "base": {"Base"}, - "computed": {"computed"}, - "inherited": {"base_only"}, - "assisted": {"kept"}, - "shared": {"both"}, - } - assert walk.met == { - (f"{prefix}table", "TABLE"): frozenset({(f"{prefix}kernel", "Kernel.run")}) - } - assert _hashed_modules(kernel_class()) == {f"{prefix}computed", f"{prefix}keyonly"} - - #: Adapters the spine build installs around a stage's own transform, each with #: why it computes no value. _SPINE_BUILD_ADAPTERS: dict[tuple[str, str], str] = { diff --git a/test_support/microcosm_build/derived_binding_rule.py b/test_support/microcosm_build/derived_binding_rule.py new file mode 100644 index 0000000000..9f4fef5442 --- /dev/null +++ b/test_support/microcosm_build/derived_binding_rule.py @@ -0,0 +1,1032 @@ +"""The derived binding rule's walk, shared by every country's graph tests. + +``source_hash`` does not follow imports, so a node's hash must name every +module whose code can change what the node computes (microcosm#931). The rule +derives those modules from what a stage's transform, or a graph kernel's own +code, names, and follows each named definition into the module that defines +it, through re-exports, lazy exports and module attributes, to a fixed point. + +The machinery here is country-neutral. A country declares where its rule +stops in a ``RuleTables`` (its core modules, refusal-only checks, kernel +stops and declaration modules), each entry with a reason its own tests +check, and walks through one ``BindingRule`` built from it. Its tests hold +each node's key to the walks. test_uk_graph.py holds the UK stages and +kernels to it; tests/engine_free/shared/test_derived_binding_rule.py holds +the controls. + +What the rule cannot see: a call through any other object whose class the +code never names, a name looked up by a computed string, and data files read +by path. +""" + +from __future__ import annotations + +import ast +import functools +import importlib +import importlib.util +import inspect +import sys +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType, ModuleType +from typing import NamedTuple + +#: The packages the rule follows: every first-party package, each under the +#: microcosm namespace (microcosm-build, -calibrate, -data, -diagnostics, +#: -fit, -frame and -graph). +FIRST_PARTY_SCOPE = "microcosm." +#: The Frame's package. A country lists the Frame's modules among its core +#: modules, and the reason it gives them covers each definition a kernel +#: names there, which kernels build and read frames with, so ``findings`` +#: reports the core definitions kernels name outside it. +FRAME_SCOPE = "microcosm.frame." +#: Packages that export names lazily, through a module ``__getattr__``, each +#: with the table, of name to the submodule defining it, that ``__getattr__`` +#: reads. The walk resolves such a name to the module that defines it and +#: follows it like any other (microcosm#931 ruling 8). microcosm.graph imports +#: its runtime modules, which need numpy, pandas and microcosm-frame, only on +#: first access. A fact about the first-party packages, not about a country: +#: test_every_lazy_export_resolves_to_the_object_python_returns checks it. +LAZY_EXPORT_TABLES: Mapping[str, str] = MappingProxyType( + {"microcosm.graph": "_RUNTIME_EXPORTS"} +) +#: First-party modules that define a module ``__getattr__`` but export +#: nothing through it, each with why. +NO_LAZY_EXPORTS: Mapping[str, str] = MappingProxyType( + { + "microcosm.build.us_runtime.sources": ( + "its __getattr__ raises for every name: it only words the error for " + "the names it removed" + ), + } +) +#: Method and function names that change their receiver or an argument in +#: place. +IN_PLACE_CALLS = frozenset( + { + "__delitem__", + "__setattr__", + "__setitem__", + "add", + "append", + "appendleft", + "at", + "clear", + "copyto", + "difference_update", + "discard", + "extend", + "extendleft", + "fill", + "fill_diagonal", + "insert", + "intersection_update", + "itemset", + "place", + "pop", + "popitem", + "put", + "putmask", + "remove", + "resize", + "reverse", + "rotate", + "setdefault", + "setflags", + "shuffle", + "sort", + "symmetric_difference_update", + "update", + } +) + + +@dataclass(frozen=True) +class RuleTables: + """One country's declaration of where its rule stops, each entry with the + reason its tests check. + + - ``core_modules``: the modules every node runs through, which the rule + neither follows nor asks a node to bind (``BindingRule.walk``). + - ``refusal_only``: checks that read values only to decide whether to + refuse, which the rule does not follow; a kernel's method is named + ``Class.method``, and the kernel rule leaves it out of what the kernel + runs (``BindingRule.kernel_body``). + - ``stops``: where the kernel rule stops by default + (``BindingRule.kernel_walk``). + - ``declaration_modules``: modules the rule neither follows nor returns, + because a node's key already hashes what it reads from them; none by + default. + - ``lazy_export_tables``: the packages' lazy exports + (``LAZY_EXPORT_TABLES`` by default). + - ``scope``: the packages the rule follows (``FIRST_PARTY_SCOPE``). + """ + + core_modules: Mapping[str, str] + refusal_only: Mapping[tuple[str, str], str] = MappingProxyType({}) + stops: Mapping[tuple[str, str], str] = MappingProxyType({}) + declaration_modules: tuple[str, ...] = () + lazy_export_tables: Mapping[str, str] = LAZY_EXPORT_TABLES + scope: str = FIRST_PARTY_SCOPE + + +class ModuleDefinitions(NamedTuple): + """What one module imports, anywhere in it, and defines at its top level.""" + + imports: dict[str, tuple[str, str]] + modules: dict[str, str] + stars: tuple[str, ...] + definitions: dict[str, object] + + +class Walk(NamedTuple): + """What a root can compute through (``BindingRule.helper_walk``).""" + + reach: dict[str, frozenset[str]] + stops: frozenset[tuple[str, str]] + core: frozenset[str] + + +class KernelWalk(NamedTuple): + """What a kernel's own code can compute through + (``BindingRule.kernel_walk``).""" + + reach: dict[str, frozenset[str]] + stops: frozenset[tuple[str, str]] + core: frozenset[str] + met: dict[tuple[str, str], frozenset] + core_names: frozenset[tuple[str, str]] + + +class KeyedWalk(NamedTuple): + """A walk the kernel rule holds a key to, and the modules that key + hashes.""" + + walk: KernelWalk + key: frozenset[str] + + +class Findings(NamedTuple): + """What a country's keyed walks find (``findings``): by label, the modules + a walk reaches that its key does not hash; the names the rule stops at + outside each key; the listed stops met; and the core definitions named + outside the Frame's modules.""" + + unhashed: dict[str, list[str]] + unlisted: frozenset[tuple[str, str]] + met: frozenset[tuple[str, str]] + core_names: frozenset[tuple[str, str]] + + +@functools.cache +def is_module(name: str) -> bool: + """Whether a first-party dotted name is a module. The rule never follows + another package, so it never imports one to find out.""" + + if not name.startswith(FIRST_PARTY_SCOPE): + return False + parent = name.rpartition(".")[0] + try: + if parent: + spec = importlib.util.find_spec(parent) + if spec is None or spec.submodule_search_locations is None: + return False + return importlib.util.find_spec(name) is not None + except (ImportError, ValueError): + return False + + +@functools.cache +def module_definitions(name: str, scope: str) -> ModuleDefinitions | None: + if not name.startswith(scope): + return None + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): + return None + if spec is None or not (spec.origin or "").endswith(".py"): + return None + package = name if spec.submodule_search_locations else name.rpartition(".")[0] + tree = ast.parse(Path(spec.origin).read_text(encoding="utf-8")) + imports: dict[str, tuple[str, str]] = {} + modules: dict[str, str] = {} + stars: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.level: + base = package.rsplit(".", node.level - 1)[0] + origin = ".".join(part for part in (base, node.module) if part) + else: + origin = node.module or "" + for alias in node.names: + if alias.name == "*": + stars.append(origin) + elif is_module(f"{origin}.{alias.name}"): + modules[alias.asname or alias.name] = f"{origin}.{alias.name}" + else: + imports[alias.asname or alias.name] = (origin, alias.name) + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.asname: + modules[alias.asname] = alias.name + else: + head = alias.name.partition(".")[0] + modules[head] = head + definitions: dict[str, object] = {} + for node in tree.body: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + definitions[node.name] = node + elif isinstance(node, ast.TypeAlias): + definitions[node.name.id] = node + elif isinstance(node, ast.Assign | ast.AnnAssign): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + for sub in ast.walk(target): + if isinstance(sub, ast.Name): + definitions[sub.id] = node + return ModuleDefinitions(imports, modules, tuple(stars), definitions) + + +def writes_nothing(node) -> bool: + """Whether a function can only refuse: it returns nothing but None, or its + sole parameter unrebound, and writes nothing: no store through a + subscript or attribute, no augmented assignment, no ``out=`` or + ``inplace=`` argument, and no call by a name that changes its receiver or + an argument in place.""" + + arguments = node.args + parameters = { + argument.arg + for argument in ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + *(a for a in (arguments.vararg, arguments.kwarg) if a is not None), + ) + } + rebound = { + sub.id + for sub in ast.walk(node) + if isinstance(sub, ast.Name) and isinstance(sub.ctx, ast.Store | ast.Del) + } + returnable = parameters - rebound if len(parameters) == 1 else set() + for sub in ast.walk(node): + if isinstance(sub, ast.Return) and sub.value is not None: + value = sub.value + if not ( + (isinstance(value, ast.Constant) and value.value is None) + or (isinstance(value, ast.Name) and value.id in returnable) + ): + return False + elif isinstance( + sub, ast.AugAssign | ast.Yield | ast.YieldFrom | ast.Global | ast.Nonlocal + ): + return False + elif isinstance(sub, ast.Assign | ast.AnnAssign | ast.Delete): + targets = ( + sub.targets + if isinstance(sub, ast.Assign | ast.Delete) + else [sub.target] + ) + if any( + isinstance(part, ast.Subscript | ast.Attribute) + for target in targets + for part in ast.walk(target) + ): + return False + elif isinstance(sub, ast.keyword) and sub.arg in {"inplace", "out"}: + return False + elif isinstance(sub, ast.Call): + function = sub.func + if isinstance(function, ast.Attribute) and function.attr in IN_PLACE_CALLS: + return False + if isinstance(function, ast.Name) and function.id in { + *IN_PLACE_CALLS, + "delattr", + "eval", + "exec", + "setattr", + }: + return False + return True + + +def string_expression(node) -> bool: + """A string literal, an f-string of names or of one call on names, or a + concatenation of them.""" + + if isinstance(node, ast.Constant): + return isinstance(node.value, str) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return string_expression(node.left) and string_expression(node.right) + if not isinstance(node, ast.JoinedStr): + return False + for part in node.values: + value = part.value if isinstance(part, ast.FormattedValue) else part + if isinstance(value, ast.Call): + if not isinstance(value.func, ast.Name) or value.keywords: + return False + value = value.args[0] if len(value.args) == 1 else None + if not isinstance(value, ast.Name | ast.Constant): + return False + return True + + +def calls_super(node) -> bool: + return any( + isinstance(sub, ast.Call) + and isinstance(sub.func, ast.Name) + and sub.func.id == "super" + for sub in ast.walk(node) + ) + + +def statement_names(statement) -> list[str]: + if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef): + return [statement.name] + if isinstance(statement, ast.Assign | ast.AnnAssign): + targets = ( + statement.targets + if isinstance(statement, ast.Assign) + else [statement.target] + ) + return [ + sub.id + for target in targets + for sub in ast.walk(target) + if isinstance(sub, ast.Name) + ] + return [] + + +def self_attributes(statement) -> set[str]: + """The attributes ``statement`` reads on ``self``, ``cls`` or + ``type(self)``.""" + + found = set() + for sub in ast.walk(statement): + if not isinstance(sub, ast.Attribute): + continue + value = sub.value + if (isinstance(value, ast.Name) and value.id in {"self", "cls"}) or ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Name) + and value.func.id == "type" + ): + found.add(sub.attr) + return found + + +def key_only_methods(body, keyed) -> frozenset[str]: + """The methods of a kernel's ``body`` that only its key computation calls: + named on ``self``, ``cls`` or ``type(self)`` by its ``keyed`` statements + (``implementation_hash``) or by another such method, and by nothing else + the kernel runs. They compute the key as the hash does.""" + + methods = { + name + for _, statement, name in body + if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef) + } + reads: dict[str, set[str]] = {} + for _, statement, name in body: + reads.setdefault(name, set()).update(self_attributes(statement)) + key = set().union(*(self_attributes(statement) for _, statement, _ in keyed)) + key_only: set[str] = set() + while True: + found = { + method + for method in methods - key_only + if method in key.union(*(reads[item] for item in key_only)) + and not any( + method in names + for other, names in reads.items() + if other != method and other not in key_only + ) + } + if not found: + return frozenset(key_only) + key_only |= found + + +def merged(first: Walk, second: Walk) -> Walk: + reach = dict(first.reach) + for module, names in second.reach.items(): + reach[module] = reach.get(module, frozenset()) | names + return Walk(reach, first.stops | second.stops, first.core | second.core) + + +def class_named(ref: tuple[str, str]) -> type: + module, name = ref + return getattr(importlib.import_module(module), name) + + +def hashed_by(compute: Callable[[], object]) -> frozenset[str]: + """Every module ``compute()`` hashes, recorded at each ``source_hash`` it + reaches, in whichever module calls it: every loaded module's + ``source_hash`` name stands in for the recorder during one call, after a + first call has imported what ``compute`` imports.""" + + from microcosm.graph import kernel as graph_kernel + + original = graph_kernel.source_hash + recorded: list[object] = [] + + def record(*objects, **dependencies): + recorded.extend(objects) + return original(*objects, **dependencies) + + compute() + patched = [ + module + for module in list(sys.modules.values()) + if isinstance(module, ModuleType) + and getattr(module, "__dict__", {}).get("source_hash") is original + ] + for module in patched: + module.source_hash = record + try: + compute() + finally: + for module in patched: + module.source_hash = original + return frozenset( + (item if isinstance(item, ModuleType) else inspect.getmodule(item)).__name__ + for item in recorded + ) + + +def hashed_modules(kernel) -> frozenset[str]: + """Every module ``kernel.implementation_hash()`` hashes (``hashed_by``).""" + + return hashed_by(kernel.implementation_hash) + + +def kernel_classes_defined(sources: Iterable[tuple[str, Path]]) -> set[type]: + """Every kernel class the given module files define: each class whose body + declares ``ref``, or that stores a ``ref`` attribute. ``sources`` pairs + each module's dotted name with its file; a module is imported only where + it defines one.""" + + defined = set() + for module, path in sources: + for node in ast.parse(Path(path).read_text(encoding="utf-8")).body: + if not isinstance(node, ast.ClassDef): + continue + declares_ref = any( + "ref" in statement_names(statement) for statement in node.body + ) or any( + isinstance(sub, ast.Attribute) + and sub.attr == "ref" + and isinstance(sub.ctx, ast.Store) + for sub in ast.walk(node) + ) + if declares_ref: + defined.add(getattr(importlib.import_module(module), node.name)) + return defined + + +def findings(walks: Mapping[str, Mapping[str, KeyedWalk]]) -> Findings: + """What the keyed walks find, by kernel ref and label (``Findings``): each + walk's modules outside its key, every name it stops at outside its key, + the listed stops it meets and the core definitions it names outside the + Frame's modules (``FRAME_SCOPE``).""" + + unhashed: dict[str, list[str]] = {} + unlisted: set[tuple[str, str]] = set() + met: set[tuple[str, str]] = set() + core_names: set[tuple[str, str]] = set() + for keyed_walks in walks.values(): + for label, (walk, key) in keyed_walks.items(): + missing = sorted(set(walk.reach) - key) + if missing: + unhashed[label] = missing + unlisted.update(item for item in walk.stops if item[0] not in key) + met |= set(walk.met) + core_names.update( + item for item in walk.core_names if not item[0].startswith(FRAME_SCOPE) + ) + return Findings( + unhashed, frozenset(unlisted), frozenset(met), frozenset(core_names) + ) + + +class BindingRule: + """The rule's walk under one country's declaration (``RuleTables``). + + Each method takes the arguments of the function it was in + test_uk_graph.py, the optional ``scope`` among them, which is the + declaration's unless given. A rule keeps its own caches of each + definition's kind and of the lazy exports, since both depend on its + tables.""" + + def __init__(self, tables: RuleTables) -> None: + self.tables = tables + self.scope = tables.scope + self._kinds: dict[tuple[str, str, str], str | None] = {} + self._lazy: dict[str, Mapping[str, str]] = {} + + def lazy_exports(self, module: str) -> Mapping[str, str]: + """Each name ``module`` exports lazily, to the module defining it, read + from the table its ``__getattr__`` reads (``lazy_export_tables``).""" + + if module not in self._lazy: + table = self.tables.lazy_export_tables.get(module) + if table is None: + self._lazy[module] = MappingProxyType({}) + else: + exports = getattr(importlib.import_module(module), table) + self._lazy[module] = MappingProxyType( + { + name: f"{module}.{submodule}" + for name, submodule in exports.items() + } + ) + return self._lazy[module] + + def resolve( + self, module: str, name: str, scope: str | None = None + ) -> tuple[str, str] | None: + """The top-level definition ``name`` refers to in ``module``, through + re-exports and lazy exports (``lazy_export_tables``); None for a + module or anything outside ``scope``.""" + + scope = self.scope if scope is None else scope + pending, seen = [(module, name)], set() + while pending: + item = pending.pop() + if item in seen: + continue + seen.add(item) + module, name = item + facts = module_definitions(module, scope) + if facts is None: + continue + if name in facts.definitions: + return item + if name in facts.imports: + pending.append(facts.imports[name]) + continue + # As in Python, the module's __getattr__ supplies a name only where + # none of its own attributes, its star imports' included, is one. + lazy = self.lazy_exports(module).get(name) + if lazy is not None: + pending.append((lazy, name)) + pending.extend((star, name) for star in facts.stars) + return None + + def named( + self, module: str, node, scope: str | None = None + ) -> set[tuple[str, str]]: + """The top-level definitions ``node`` names, annotations included: a + bare name, or a name read by attribute on a module, each resolved + through re-exports.""" + + scope = self.scope if scope is None else scope + facts = module_definitions(module, scope) + found = set() + for sub in ast.walk(node): + if isinstance(sub, ast.Name): + hit = self.resolve(module, sub.id, scope) + elif isinstance(sub, ast.Attribute): + chain = [] + value = sub + while isinstance(value, ast.Attribute): + chain.insert(0, value.attr) + value = value.value + target = ( + facts.modules.get(value.id) if isinstance(value, ast.Name) else None + ) + while target and chain and is_module(f"{target}.{chain[0]}"): + target = f"{target}.{chain.pop(0)}" + hit = ( + self.resolve(target, chain[0], scope) if target and chain else None + ) + else: + continue + if hit is not None: + found.add(hit) + return found + + def helper_kind(self, module: str, name: str, scope: str | None = None) -> str: + """How a top-level definition can bear on a value. + + ``value`` unless it is a listed refusal-only ``check`` + (``refusal_only``) or provably one of: a ``name`` (one string: a + literal, an f-string or concatenation of names, or another name of + these kinds); a ``column name`` (a function returning one such string + built from its arguments); a ``record`` (a class of undefaulted fields + whose types are of these kinds, no methods). A collection of strings + is a value, since it selects rows, columns or codes, as is a number. A + cycle reads as ``value``. + """ + + scope = self.scope if scope is None else scope + if (module, name) in self.tables.refusal_only: + return "check" + key = (module, name, scope) + if key in self._kinds: + return self._kinds[key] or "value" + self._kinds[key] = None + self._kinds[key] = self._classify(module, name, scope) + return self._kinds[key] + + def _names_only_harmless(self, module: str, node, scope: str) -> bool: + return all( + found[0].startswith(self.tables.declaration_modules) + or self.helper_kind(*found, scope) != "value" + for found in self.named(module, node, scope) + ) + + def _classify(self, module: str, name: str, scope: str) -> str: + node = module_definitions(module, scope).definitions[name] + # A type alias statement (``type X = ...``) reads as an assignment of + # its value. + if isinstance(node, ast.Assign | ast.AnnAssign | ast.TypeAlias): + if node.value is None or ( + ( + isinstance(node.value, ast.Name | ast.Attribute) + or string_expression(node.value) + ) + and self._names_only_harmless(module, node.value, scope) + ): + return "name" + return "value" + if isinstance(node, ast.ClassDef): + fields = [item for item in node.body if isinstance(item, ast.AnnAssign)] + fields_only = all( + (isinstance(item, ast.Expr) and isinstance(item.value, ast.Constant)) + or isinstance(item, ast.Pass) + or (isinstance(item, ast.AnnAssign) and item.value is None) + for item in node.body + ) + if fields_only and all( + self._names_only_harmless(module, item, scope) + for item in ( + *node.bases, + *node.keywords, + *node.decorator_list, + *(field.annotation for field in fields), + ) + ): + return "record" + return "value" + body = list(node.body) + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + ): + body = body[1:] + if ( + len(body) == 1 + and isinstance(body[0], ast.Return) + and string_expression(body[0].value) + and self._names_only_harmless(module, body[0].value, scope) + ): + return "column name" + return "value" + + def helper_walk( + self, + root: tuple[str, str], + scope: str | None = None, + *, + follow_core: bool = False, + ) -> Walk: + """What ``root`` can compute through, where the rule stops, and the + core modules it meets. + + ``reach`` maps every module holding a definition that can change what + ``root`` computes to those definitions: ``root`` itself, and each + ``value`` definition it names, and they name, to a fixed point. + ``stops`` is every ``name``, ``column name`` or ``record`` those + definitions name, which the rule does not follow. ``core`` is every + listed core module (``core_modules``) whose definitions they name, + which the rule neither follows nor returns in ``reach``, unless + ``follow_core``. The declaration modules and the listed refusal-only + checks are neither followed nor returned.""" + + scope = self.scope if scope is None else scope + module, name = root + node = module_definitions(module, scope).definitions[name] + walk, _, _ = self.walk( + {root}, + [(found, root) for found in self.named(module, node, scope)], + scope, + follow_core=follow_core, + ) + return walk + + def walk( + self, + roots: set[tuple[str, str]], + seeds: list[tuple[tuple[str, str], object]], + scope: str | None = None, + *, + follow_core: bool = False, + halts: Mapping[tuple[str, str], str] = MappingProxyType({}), + ) -> tuple[Walk, dict[tuple[str, str], frozenset], frozenset[tuple[str, str]]]: + """The walk behind ``helper_walk`` and ``kernel_walk``: ``roots`` are + reached as given, and each ``(definition, namer)`` seed is followed as + ``helper_walk`` follows what its root names. A definition in ``halts`` + is neither followed nor returned; the second result maps each one met + to what named it, and the third holds every core module's definition + met.""" + + scope = self.scope if scope is None else scope + declarations = self.tables.declaration_modules + core_modules = self.tables.core_modules + reached: dict[str, set[str]] = {} + for module, name in roots: + reached.setdefault(module, set()).add(name) + stops: set[tuple[str, str]] = set() + core: set[str] = set() + core_names: set[tuple[str, str]] = set() + met: dict[tuple[str, str], set] = {} + pending, seen = list(seeds), set(roots) + while pending: + item, namer = pending.pop() + if item in halts: + met.setdefault(item, set()).add(namer) + continue + if item in seen: + continue + seen.add(item) + module, name = item + if module.startswith(declarations): + continue + if module in core_modules: + core.add(module) + core_names.add(item) + if not follow_core: + continue + kind = self.helper_kind(module, name, scope) + if kind != "value": + if kind != "check": + stops.add(item) + continue + reached.setdefault(module, set()).add(name) + node = module_definitions(module, scope).definitions[name] + pending.extend((found, item) for found in self.named(module, node, scope)) + walk = Walk( + {module: frozenset(names) for module, names in reached.items()}, + frozenset(stops), + frozenset(core), + ) + return ( + walk, + {item: frozenset(namers) for item, namers in met.items()}, + frozenset(core_names), + ) + + def helper_reach( + self, root: tuple[str, str], scope: str | None = None + ) -> dict[str, frozenset[str]]: + """Every module holding a definition that can change what ``root`` + computes, with those definitions (``helper_walk``).""" + + return self.helper_walk(root, scope).reach + + def kernel_classes(self, kernel_class: type) -> list[type]: + """The first-party classes a kernel's code comes from, in its method + resolution order, without the core modules' (``KernelBase``).""" + + return [ + cls + for cls in kernel_class.__mro__ + if cls.__module__.startswith(self.scope) + and cls.__module__ not in self.tables.core_modules + ] + + def kernel_body(self, kernel_class: type) -> list[tuple[type, object, str]]: + """What a kernel of ``kernel_class`` runs: each method or class + attribute with its class and name, from the first class in the method + resolution order that defines it, as Python resolves it, and a base's + definition too where the one that wins calls ``super()``. + ``implementation_hash`` is left out, since it computes the key: an + edit to what it calls that changes what it returns changes the key. So + is a method only the key computation calls (``key_only_methods``), + such as the UK full gates' ``_country_resources``, and so are the + listed refusal-only methods.""" + + chosen: dict[str, object | None] = {} + body = [] + keyed = [] + for cls in self.kernel_classes(kernel_class): + definitions = module_definitions(cls.__module__, self.scope).definitions + for statement in definitions[cls.__name__].body: + for name in statement_names(statement): + if name in chosen: + previous = chosen[name] + if previous is not None and calls_super(previous): + chosen[name] = statement + (keyed if name == "implementation_hash" else body).append( + (cls, statement, name) + ) + continue + if ( + cls.__module__, + f"{cls.__name__}.{name}", + ) in self.tables.refusal_only: + chosen[name] = None + continue + chosen[name] = statement + (keyed if name == "implementation_hash" else body).append( + (cls, statement, name) + ) + key_only = key_only_methods(body, keyed) + return [item for item in body if item[2] not in key_only] + + def kernel_walk( + self, + kernel_class: type, + scope: str | None = None, + *, + halts: Mapping[tuple[str, str], str] | None = None, + ) -> KernelWalk: + """What a kernel's own code can compute through: the rule followed + from what ``kernel_body`` names, as from a stage's transform, stopping + also at ``halts`` (``stops`` unless given). Its classes are in + ``reach``; ``met`` maps each listed stop it meets to what names it + there (a definition, or ``Class.attribute`` for the kernel's own + code), and ``core_names`` holds the core modules' definitions it + names.""" + + scope = self.scope if scope is None else scope + classes = self.kernel_classes(kernel_class) + seeds = [ + (found, (cls.__module__, f"{cls.__name__}.{name}")) + for cls, statement, name in self.kernel_body(kernel_class) + for found in self.named(cls.__module__, statement, scope) + ] + walk, met, core_names = self.walk( + {(cls.__module__, cls.__name__) for cls in classes}, + seeds, + scope, + halts=self.tables.stops if halts is None else halts, + ) + return KernelWalk(walk.reach, walk.stops, walk.core, met, core_names) + + def class_attributes(self, cls: type) -> dict[str, list[tuple[type, object]]]: + """Each attribute ``cls`` defines, with the class and statement that + define it, as ``kernel_body`` chooses them by method resolution + order.""" + + body: dict[str, list[tuple[type, object]]] = {} + for owner, statement, name in self.kernel_body(cls): + body.setdefault(name, []).append((owner, statement)) + return body + + def engine_methods(self, engine_class: type, walk: Walk) -> frozenset[str]: + """The attributes of ``engine_class`` that the code ``walk`` reaches + names: read by attribute, on any receiver, or through ``getattr`` or + ``hasattr`` with a constant name. It matches by name alone, so it sees + the engine under any name the code holds it by, and can only + over-count.""" + + found: set[object] = set() + for module, names in walk.reach.items(): + definitions = module_definitions(module, self.scope).definitions + for name in names: + for sub in ast.walk(definitions[name]): + if isinstance(sub, ast.Attribute): + found.add(sub.attr) + elif ( + isinstance(sub, ast.Call) + and isinstance(sub.func, ast.Name) + and sub.func.id in {"getattr", "hasattr"} + and len(sub.args) > 1 + and isinstance(sub.args[1], ast.Constant) + ): + found.add(sub.args[1].value) + return frozenset(found & set(self.class_attributes(engine_class))) + + def engine_roots(self, engine_class: type) -> frozenset[tuple[str, str]]: + """The classes an engine of ``engine_class`` comes from, the roots of + its walk. Their definitions name every method, so the engine methods a + walk names are read outside them (``engine_reach``).""" + + return frozenset( + (cls.__module__, cls.__name__) for cls in self.kernel_classes(engine_class) + ) + + def engine_seeds( + self, + engine_class: type, + methods: frozenset[str], + scope: str | None = None, + ) -> list[tuple[tuple[str, str], tuple[str, str]]]: + """What ``methods`` of ``engine_class``, the constructor the builds call + and every method they call on ``self`` name, each with the method + naming it: the seeds of the engine walk (``engine_walk``).""" + + scope = self.scope if scope is None else scope + body = self.class_attributes(engine_class) + pending, chosen = sorted({"__init__", *methods} & set(body)), set() + while pending: + name = pending.pop() + if name in chosen: + continue + chosen.add(name) + for _, statement in body[name]: + pending.extend( + sub.attr + for sub in ast.walk(statement) + if isinstance(sub, ast.Attribute) + and isinstance(sub.value, ast.Name) + and sub.value.id in {"self", "cls"} + and sub.attr in body + ) + return [ + (found, (owner.__module__, f"{owner.__name__}.{name}")) + for name in sorted(chosen) + for owner, statement in body[name] + for found in self.named(owner.__module__, statement, scope) + ] + + def engine_walk( + self, + engine_class: type, + methods: frozenset[str], + scope: str | None = None, + ) -> Walk: + """What an engine of ``engine_class`` computes through for code that + calls ``methods`` on it: the rule's walk, at ``scope`` with the same + tables, from those methods, the constructor the builds call, and every + method they call on ``self``, to a fixed point.""" + + scope = self.scope if scope is None else scope + walk, _, _ = self.walk( + set(self.engine_roots(engine_class)), + self.engine_seeds(engine_class, methods, scope), + scope, + ) + return walk + + def engine_methods_beyond(self, engine_class: type, walk: Walk) -> frozenset[str]: + """The methods of ``engine_class`` that ``walk`` names outside the + engine's own classes (``engine_methods``, ``engine_roots``).""" + + roots = self.engine_roots(engine_class) + return self.engine_methods( + engine_class, + Walk( + { + module: frozenset( + name for name in names if (module, name) not in roots + ) + for module, names in walk.reach.items() + }, + walk.stops, + walk.core, + ), + ) + + def engine_reach( + self, engine_class: type, walk: Walk, scope: str | None = None + ) -> Walk: + """What an engine of ``engine_class`` computes through for code whose + walk is ``walk``: the engine walk (``engine_walk``) from the methods + ``walk`` names, and from those the engine walk's own reach names on + any receiver outside the engine's classes, to a fixed point. So a + helper the engine hands itself to, which calls another of its methods + on that argument, is followed into that method too.""" + + methods = self.engine_methods(engine_class, walk) + while True: + reached = self.engine_walk(engine_class, methods, scope) + more = methods | self.engine_methods_beyond(engine_class, reached) + if more == methods: + return reached + methods = more + + def refusal_only_violations(self) -> list[tuple[str, str]]: + """The listed refusal-only checks (``refusal_only``) whose definition + can return a value or write (``writes_nothing``).""" + + violations = [] + for module, name in self.tables.refusal_only: + owner, _, method = name.rpartition(".") + node = module_definitions(module, self.scope).definitions[owner or name] + if owner: + (node,) = [ + item + for item in node.body + if isinstance(item, ast.FunctionDef) and item.name == method + ] + if not writes_nothing(node): + violations.append((module, name)) + return sorted(violations) + + def core_modules_off_run_graph_path(self) -> list[str]: + """The listed core modules (``core_modules``) the walk from run_graph, + which keys, runs and patches every node, does not meet.""" + + executor = self.helper_walk( + ("microcosm.graph.executor", "run_graph"), follow_core=True + ) + on_path = set(executor.reach) | set(executor.core) + return sorted(set(self.tables.core_modules) - on_path) From 1ed4ada81b89c58b65262a4552906a18bf13b39d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 04:36:05 +0000 Subject: [PATCH 045/108] Count a store into a container a function creates as local in the refusal check (#931) The refusal check (writes_nothing) refused every store through a subscript. So a check that records what it has seen in a dict of its own could not be listed as refusal-only: uk_runtime.ledger_targets._assert_household_type_bindings_declared, which stores covered[value] = target_id, is one. This commit teaches the check that such a container is the function's own. It changes the shared machinery and its controls only. - local_containers finds them. A name is local when: - the function's own scope binds it to a new container: a dict, list or set display or comprehension, or a call of dict, list or set; - every other binding of it anywhere in the function, nested scopes included, is one too; - it is no parameter of the function or of anything nested in it. Bindings of every kind count against it: loop, with, walrus, unpacking and comprehension targets, deletions, imports, nested definitions, and exception and match captures. A call counts only where the function binds no dict, list or set of its own. - writes_nothing now lets the function store into or delete one item of a local container, call an in-place method on it, or augment it as a whole, in its own body or in a closure. It still refuses: - a nested store, into an item of an item or an attribute of one; - an attribute store on the container; - a store through any other name, including an alias of the container or of a parameter; - returning the container. - Two departures from the design: - An augmented store into an item (counts[row] += 1) is still a write. It calls the item's own augmented operator, which changes a list or array item in place, and the item may be an object the function was handed: seen = list(rows) holds the rows' own objects. The plain store counts[row] = counts.get(row, 0) + 1 is allowed. - The fresh binding must be in the function's own scope. The design also counted one in a nested scope, but a name that only a nested function binds is a global in the function itself. - A new control holds 10 functions that write only into their own containers and 14 that still write. What moves: nothing. No table, walk, key, value, declaration or generated file moves, since writes_nothing only checks the listed refusal-only entries. This commit neither lists _assert_household_type_bindings_declared nor unbinds frs_relationships; the UK follow-up does both. Tests: all code facts. - _assert_household_type_bindings_declared fails the check at 8ddd0a63c and passes it here (checked in scratch). - The six UK refusal-only entries still pass, and the scratch dump of every UK walk is unchanged. - Of the design prototype's 27 controls, all but its augmented-item case keep their verdict. - test_derived_binding_rule (6 tests) and test_uk_graph (48) pass. Ruff is clean. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_derived_binding_rule.py | 125 ++++++++++++ .../microcosm_build/derived_binding_rule.py | 183 ++++++++++++++++-- 2 files changed, 294 insertions(+), 14 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py b/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py index 361a891d50..614354dd10 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py @@ -25,6 +25,7 @@ BindingRule, RuleTables, hashed_modules, + local_containers, module_definitions, writes_nothing, ) @@ -60,6 +61,130 @@ def test_the_refusal_check_sees_each_way_a_function_writes() -> None: assert writes_nothing(function) is (label in refusing), label +def test_the_refusal_check_lets_a_function_write_into_containers_it_creates(): + """Control for ``local_containers`` (microcosm#931): a function that + stores into, deletes from, augments or calls an in-place method on a + container its own scope creates, and that no other binding reaches, + writes nothing outside itself, a closure over it included. A store into + an item of an item or an attribute of one, an augmented store into an + item (``seen = list(rows)`` holds the rows' own objects), a store through + a name bound otherwise anywhere in the function (an alias, a parameter, a + loop target or an unpacked binding), one through a name a nested scope + binds, which in the function is a global, a container built by a + ``dict`` the function binds, and returning the container still count.""" + refusing = { + "local dict": ( + "def f(rows):\n" + " seen = {}\n" + " for row in rows:\n" + " if row in seen:\n" + " raise ValueError('x')\n" + " seen[row] = 1\n" + ), + "annotated local dict": ( + "def f(rows):\n" + " seen: dict[str, int] = {}\n" + " for row in rows:\n" + " seen[row] = 1\n" + " if len(seen) != len(rows):\n" + " raise ValueError('x')\n" + ), + "local list append": ( + "def f(rows):\n" + " bad = []\n" + " for row in rows:\n" + " if row < 0:\n" + " bad.append(row)\n" + " if bad:\n" + " raise ValueError(bad)\n" + ), + "local set add": ( + "def f(rows):\n seen = set()\n for row in rows:\n seen.add(row)\n" + ), + "local counter": ( + "def f(rows):\n" + " counts = dict()\n" + " for row in rows:\n" + " counts[row] = counts.get(row, 0) + 1\n" + ), + "local list augmented": ( + "def f(rows):\n" + " bad = []\n" + " bad += [row for row in rows if row < 0]\n" + " if bad:\n" + " raise ValueError(bad)\n" + ), + "local item deleted": ( + "def f(rows):\n" + " seen = {row: 1 for row in rows}\n" + " del seen[rows[0]]\n" + " if seen:\n" + " raise ValueError('x')\n" + ), + "local slice": "def f(rows):\n window = list(rows)\n window[1:] = []\n", + "local unpacked items": ( + "def f(rows):\n pair = [0, 0]\n pair[0], pair[1] = rows\n" + ), + "closure over a local": ( + "def f(rows):\n" + " seen = set()\n\n" + " def note(row):\n" + " seen.add(row)\n\n" + " for row in rows:\n" + " note(row)\n" + ), + } + writing = { + "alias of a parameter": "def f(values):\n seen = values\n seen[0] = 1\n", + "rebound to a parameter": ( + "def f(values):\n seen = {}\n seen = values\n seen['a'] = 1\n" + ), + "nested store": ( + "def f(values):\n seen = {'a': values}\n seen['a'][0] = 1\n" + ), + "item's method": "def f(values):\n seen = [values]\n seen[0].append(1)\n", + "augmented item": "def f(rows):\n seen = list(rows)\n seen[0] += [1]\n", + "attribute on a local": "def f(rows):\n seen = {}\n seen.a = 1\n", + "unpacked into a parameter": ( + "def f(values):\n seen = [0]\n seen[0], values[0] = 1, 2\n" + ), + "returns the local": ( + "def f(rows):\n seen = {}\n seen['a'] = 1\n return seen\n" + ), + "shadowed dict": "def f(dict):\n seen = dict()\n seen['a'] = 1\n", + "parameter named like a local": ( + "def f(seen):\n seen = {}\n seen['a'] = 1\n" + ), + "unpacked binding": ( + "def f(values):\n seen, other = {}, values\n other[0] = 1\n" + ), + "loop target": ( + "def f(rows):\n" + " seen = []\n" + " for seen in rows:\n" + " seen.append(1)\n" + ), + "nested parameter": ( + "def f(rows):\n" + " seen = {}\n\n" + " def note(seen):\n" + " seen['a'] = 1\n\n" + " note(rows)\n" + ), + "bound only in a nested scope": ( + "def f(rows):\n def note():\n seen = {}\n\n seen['a'] = 1\n" + ), + } + for label, source in {**refusing, **writing}.items(): + function = ast.parse(source).body[0] + assert writes_nothing(function) is (label in refusing), label + # The containers it counts as the function's own. + function = ast.parse(refusing["closure over a local"]).body[0] + assert local_containers(function) == {"seen"} + function = ast.parse(writing["unpacked binding"]).body[0] + assert local_containers(function) == frozenset() + + def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( tmp_path, monkeypatch ) -> None: diff --git a/test_support/microcosm_build/derived_binding_rule.py b/test_support/microcosm_build/derived_binding_rule.py index 9f4fef5442..60a2611f62 100644 --- a/test_support/microcosm_build/derived_binding_rule.py +++ b/test_support/microcosm_build/derived_binding_rule.py @@ -102,6 +102,9 @@ } ) +#: The builtins whose call returns a new container (``local_containers``). +FRESH_CONTAINER_CALLS = frozenset({"dict", "list", "set"}) + @dataclass(frozen=True) class RuleTables: @@ -249,15 +252,9 @@ def module_definitions(name: str, scope: str) -> ModuleDefinitions | None: return ModuleDefinitions(imports, modules, tuple(stars), definitions) -def writes_nothing(node) -> bool: - """Whether a function can only refuse: it returns nothing but None, or its - sole parameter unrebound, and writes nothing: no store through a - subscript or attribute, no augmented assignment, no ``out=`` or - ``inplace=`` argument, and no call by a name that changes its receiver or - an argument in place.""" - +def _parameters(node) -> set[str]: arguments = node.args - parameters = { + return { argument.arg for argument in ( *arguments.posonlyargs, @@ -266,12 +263,161 @@ def writes_nothing(node) -> bool: *(a for a in (arguments.vararg, arguments.kwarg) if a is not None), ) } + + +def _fresh_container(value, calls: frozenset[str]) -> bool: + """A new container: a dict, list or set display or comprehension, or a + call by one of the names ``calls``.""" + + if isinstance( + value, ast.Dict | ast.List | ast.Set | ast.DictComp | ast.ListComp | ast.SetComp + ): + return True + return ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Name) + and value.func.id in calls + ) + + +def _own_scope(node) -> set[int]: + """The nodes of ``node``'s own scope, by id: everything in it but the + functions, lambdas and classes nested in it.""" + + found: set[int] = set() + pending = list(ast.iter_child_nodes(node)) + while pending: + sub = pending.pop() + found.add(id(sub)) + if not isinstance( + sub, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda | ast.ClassDef + ): + pending.extend(ast.iter_child_nodes(sub)) + return found + + +def local_containers(node) -> frozenset[str]: + """The names a function binds only to containers it creates itself. + + A name is local when the function's own scope binds it to a new + container (``name = `` or ``name: T = ``, where fresh is a + dict, list or set display or comprehension, or a call of ``dict``, + ``list`` or ``set``), every other binding of it anywhere in the function, + nested scopes included, is one too, and it is a parameter of neither the + function nor anything nested in it. A bare annotation binds nothing, and + an augmented assignment to the name itself keeps it local, since a + container's augmented operators change it in place or refuse. A call + counts only where the function binds no name ``dict``, ``list`` or + ``set`` of its own; one its module binds at its top level is not seen.""" + + parameters: set[str] = set() + for sub in ast.walk(node): + if isinstance(sub, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda): + parameters |= _parameters(sub) + # Each single-name binding with its value, and the name targets they and + # the bindings of no new object (a bare annotation, an augmented + # assignment to a name) account for. + valued: dict[int, tuple[str, ast.expr]] = {} + accounted: set[int] = set() + for sub in ast.walk(node): + if ( + isinstance(sub, ast.Assign) + and len(sub.targets) == 1 + and isinstance(sub.targets[0], ast.Name) + ): + valued[id(sub)] = (sub.targets[0].id, sub.value) + accounted.add(id(sub.targets[0])) + elif isinstance(sub, ast.AnnAssign) and isinstance(sub.target, ast.Name): + if sub.value is not None: + valued[id(sub)] = (sub.target.id, sub.value) + accounted.add(id(sub.target)) + elif isinstance(sub, ast.AugAssign) and isinstance(sub.target, ast.Name): + accounted.add(id(sub.target)) + # Every other binding: loop, with, walrus, unpacking and comprehension + # targets, deletions, imports, nested definitions, exception and match + # captures. + other: set[str] = set() + for sub in ast.walk(node): + if isinstance(sub, ast.Name) and isinstance(sub.ctx, ast.Store | ast.Del): + if id(sub) not in accounted: + other.add(sub.id) + elif isinstance(sub, ast.alias): + other.add((sub.asname or sub.name).partition(".")[0]) + elif ( + isinstance(sub, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) + and sub is not node + ): + other.add(sub.name) + elif isinstance(sub, ast.ExceptHandler | ast.MatchAs | ast.MatchStar): + if sub.name is not None: + other.add(sub.name) + elif isinstance(sub, ast.MatchMapping) and sub.rest is not None: + other.add(sub.rest) + stored = { + sub.id + for sub in ast.walk(node) + if isinstance(sub, ast.Name) and isinstance(sub.ctx, ast.Store | ast.Del) + } + calls = FRESH_CONTAINER_CALLS - (parameters | other | stored) + own = _own_scope(node) + fresh = { + name + for key, (name, value) in valued.items() + if key in own and _fresh_container(value, calls) + } + stale = { + name for name, value in valued.values() if not _fresh_container(value, calls) + } + return frozenset(fresh - stale - other - parameters) + + +def _stored(target): + """The parts of an assignment or deletion target that receive a value or + lose one: the target, or each element of a tuple or list target, starred + or not, at any depth.""" + + if isinstance(target, ast.Tuple | ast.List): + for element in target.elts: + yield from _stored(element) + elif isinstance(target, ast.Starred): + yield from _stored(target.value) + else: + yield target + + +def _item_of(target, local: frozenset[str]) -> bool: + """``target`` is one item of a local container: ``name[...]``.""" + + return ( + isinstance(target, ast.Subscript) + and isinstance(target.value, ast.Name) + and target.value.id in local + ) + + +def writes_nothing(node) -> bool: + """Whether a function can only refuse: it returns nothing but None, or its + sole parameter unrebound, and writes nothing outside itself: no store + through a subscript or attribute, no augmented assignment, no ``out=`` or + ``inplace=`` argument, and no call by a name that changes its receiver or + an argument in place. + + A container the function creates is its own (``local_containers``): it + may store into or delete one item of it, call an in-place method on it, + or augment it as a whole. A nested store (into an item of an item, or an + attribute of one), an augmented store into an item, which can change in + place an object the function was handed, and a store through any other + name, an alias of the container included, are still writes, and + returning the container returns a value.""" + + parameters = _parameters(node) rebound = { sub.id for sub in ast.walk(node) if isinstance(sub, ast.Name) and isinstance(sub.ctx, ast.Store | ast.Del) } returnable = parameters - rebound if len(parameters) == 1 else set() + local = local_containers(node) for sub in ast.walk(node): if isinstance(sub, ast.Return) and sub.value is not None: value = sub.value @@ -280,9 +426,10 @@ def writes_nothing(node) -> bool: or (isinstance(value, ast.Name) and value.id in returnable) ): return False - elif isinstance( - sub, ast.AugAssign | ast.Yield | ast.YieldFrom | ast.Global | ast.Nonlocal - ): + elif isinstance(sub, ast.AugAssign): + if not (isinstance(sub.target, ast.Name) and sub.target.id in local): + return False + elif isinstance(sub, ast.Yield | ast.YieldFrom | ast.Global | ast.Nonlocal): return False elif isinstance(sub, ast.Assign | ast.AnnAssign | ast.Delete): targets = ( @@ -291,16 +438,24 @@ def writes_nothing(node) -> bool: else [sub.target] ) if any( - isinstance(part, ast.Subscript | ast.Attribute) + isinstance(inner, ast.Subscript | ast.Attribute) for target in targets - for part in ast.walk(target) + for part in _stored(target) + if not _item_of(part, local) + for inner in ast.walk(part) ): return False elif isinstance(sub, ast.keyword) and sub.arg in {"inplace", "out"}: return False elif isinstance(sub, ast.Call): function = sub.func - if isinstance(function, ast.Attribute) and function.attr in IN_PLACE_CALLS: + if ( + isinstance(function, ast.Attribute) + and function.attr in IN_PLACE_CALLS + and not ( + isinstance(function.value, ast.Name) and function.value.id in local + ) + ): return False if isinstance(function, ast.Name) and function.id in { *IN_PLACE_CALLS, From c45f6ed7f980dc11991e25aafc48e6eac6806777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:32:41 +0000 Subject: [PATCH 046/108] Hold every transport kernel to the derived binding rule (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On 9 Nov María asked for the bigger transport change: hold every transport kernel to the derived binding rule, as #931 did for the UK, instead of only adding the canonical encoder to the transport compile list. This commit does so with the transport's own tables, and binds what the rule finds. It moves keys only. - test_transport_binding_rule holds the 45 kernels build_transport_registry registers: the transport package's 37 kernel classes, the shared atomic geography's four, calibrate.ordered_adam@1 and the three rules kernels. test_support/microcosm_build/transport_kernels.py replays the registry's registration calls with a stand-in AxiomEngine, and the test holds the replay to a real build_transport_registry, built engine-free with the toy package's engines. Each table entry has a reason a test checks: - the eight core modules, each on run_graph's path, with transport reasons (the transport graph tests run composed graphs through run_graph); no transport kernel declares SeedSource.EXECUTOR; - 19 core definitions: 15 graph records, enums and constants, and four with transport reasons; - two refusal-only checks, graph_inputs.require_outputs and reference_profiles._validate_interval_resolution, both of which writes_nothing passes; target compilation gives the second no interval document; - one stop, target_kernels._COMPILE_MODULES, met only through the first quantile map's class and read only inside implementation hashes; - no unbound names and no declaration modules; - what the registry hands kernels whose classes their code need not name: the AxiomEngine behind the three rules kernels and the input closure, and TRANSPORT_GATE_REGISTRY behind the battery, both checked against build_transport_registry's code; and the battery's bindings' source closures, which its key binds beyond its source_hash calls (checked: a changed digest re-keys it). - The shared module gains two things: - keyed_walk, which walks a kernel with what its build hands it. Each handed object is walked on its own, with the kernel's halts, and joins the kernel's walk, so it can only add to what the kernel walk finds, as in the design's prototype. One walk with the engine as one more root would stop following the AxiomEngine class whole where the input closure names it for an isinstance check, and so miss frame.materialize. - GRAPH_RECORD_REASONS, the graph-fact reasons. The UK's _KERNEL_CORE_NAMES takes its eight from there and keeps its 12 keys; ArtifactOutput's text now reads for any kernel. - What is bound: 79 (kernel, module) pairs in 36 kernels, in 14 files, each with a short comment on what computes through it: - the canonical encoder in 32 kernels. The seven that hash the compile list gain it through the list. In ten (the four column kernels, the measure, two money kernels, two reporting kernels and the currency) only canonical_document_param's spelling check computes through it; - reference_factors, ledger_artifact and chronicle_epoch, through which decode_target_surface or validate_reference_resolution checks a surface's factor evidence and its Chronicle feed identity, in targets.problem, takeup.assign, diagnostics.calibration and geography.support_from_facts. calibration.prepare, which hashed reference_factors already, gains the other two, and the battery gains reference_factors alone; - artifact_types in support_from_facts, takeup.assign, takeup.cell_diagnostics and the three reporting kernels; calibrate.target in support_from_facts and takeup.assign; graph_atomic_geography in support_from_facts; - country_spec, gate_battery and gates in export.prepare and transport.package; build.trace in export.readback; takeup_kernels in the battery; - the Axiom input surface in concepts.encode and encode_groups, and so scenario_override; frame.rules and frame.materialize in the input closure; calibrate.hierarchy in the weights audit and the diagnostics, which also gain monetary_binding; - frame.concepts in targets.problem, for one name, the source household column the interval diagnostics count by; - frame.exact_money in the counterfactual bridge, through a new implementation_hash that the zero solve now inherits unchanged but for the encoder; - the ordered solve's internals (exact_k, the L0 gates, their initialization record, the hierarchy, monetary_binding and the encoder) in calibrate.ordered_adam@1's adapter half. calibrate.adam@1's hash, which UK kernels and microcosm-graph's parity pins embed, does not move. - compile_modules.py. The encoder joins TRANSPORT_COMPILE_MODULE_NAMES (now 14 modules) and leaves EXCLUDED_DIRECT_IMPORTS. _GRAPH_RUNTIME now covers only the kernel protocol and the declarations, with why: both are core modules the rule lists, and binding either would re-key every node. The docstring loses the #1497 sentence. No kernel hashes the file. - test_transport_compile_modules drops UNCLASSIFIED_IMPORTS, the kernel groups and the record check, as 08b437a8e did for the UK. It keeps the closure checks, the geography-label re-key test and HASH_INPUTS. Its list test becomes test_no_transport_kernel_hashes_a_hash_input: the list is still imported where listed, exactly the seven listed kernels hash its modules, and none of the 45 hashes the list itself. first_party_imports' recorded_import_mismatches and unhashed_imports, which nothing else used, go with their two controls. - The pinned compositions follow: test_transport_evidence_kernels' module lists, and the by-ref kernel's three in microcosm-frame's test_rules_kernels. - transport_composed gains toy_rules_bindings, the bindings document make_composed_fixture built inline, which the new test uses too. - The changelog fragment 931-transport-helper-bindings says what moves. What moves: node keys only. No value, declaration or generated file moves. - 40 of the 45 transport kernel hashes move. The four atomic geography kernels and geography.person_location@1 keep theirs. - New Zealand's packaged graph, composed as test_nz_transport_declarations composes it: all 357 nodes re-key, 353 directly and four (nz.geo.assign, derive, gate and person) by chaining. Belgium composes no graph. - 0 of the 99 UK kernel hashes move, calibrate.adam@1 keeps f6554bfe..., simulate.rules@1's hashed files are untouched, and nothing of the US's moves. No committed file records a transport kernel hash or node key. Tests: all code facts. - A scratch dump of every UK walk the rule computes, with the 99 UK kernel hashes, at 8ddd0a63c and at this commit, differs only in ArtifactOutput's reason text. - Six mutations of a scratch copy are caught: a dropped binding (the column kernels' encoder), a new unhashed helper in a money kernel's run, a reader of the compile list outside a hash, an AxiomEngine method computing through a new module, the battery's binding sources left out of its key, and a rules kernel left out of the handed-engine table. - On 8ddd0a63c's sources the new test fails, on the 77 unhashed (kernel, module) pairs in 36 kernels; 26 stop names lie outside their keys there. - The 54 engine-free test files that import a changed module pass: 4,268 tests, with 105 skipped where a Belgian or NZ Chronicle artifact, the Axiom native extension or a pinned-source checkout is absent. Two failed in a first run, from a source edit made while the run held a battery kernel, which then refuses its changed binding registry; both files pass on a rerun (500 tests). test_graph_parity_pins passes (11 tests). Ruff is clean, and the test plan verifies. Co-Authored-By: Claude Opus 5.5 --- .../931-transport-helper-bindings.changed.md | 11 + .../build/transport/column_kernels.py | 32 +- .../build/transport/compile_modules.py | 31 +- .../build/transport/evidence_contract.py | 9 + .../microcosm/build/transport/gate_kernels.py | 6 + .../build/transport/geography_kernels.py | 14 +- .../build/transport/measure_kernels.py | 4 + .../build/transport/money_kernels.py | 11 + .../build/transport/population_kernels.py | 4 + .../build/transport/reporting_kernels.py | 8 +- .../build/transport/takeup_kernels.py | 21 +- .../build/transport/target_kernels.py | 17 + .../build/transport/terminal_kernels.py | 31 + .../shared/test_first_party_imports.py | 69 +-- .../shared/test_transport_binding_rule.py | 548 +++++++++++++++++ .../shared/test_transport_compile_modules.py | 578 ++---------------- .../shared/test_transport_evidence_kernels.py | 6 + .../tests/engine_free/uk/test_uk_graph.py | 67 +- .../microcosm/calibrate/ordered_kernels.py | 22 +- .../src/microcosm/frame/rules_kernels.py | 54 +- .../engine_free/shared/test_rules_kernels.py | 4 + .../microcosm_build/derived_binding_rule.py | 164 +++++ .../microcosm_build/first_party_imports.py | 98 +-- .../microcosm_build/transport_composed.py | 25 +- .../microcosm_build/transport_kernels.py | 64 ++ 25 files changed, 1109 insertions(+), 789 deletions(-) create mode 100644 changelog.d/931-transport-helper-bindings.changed.md create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_transport_binding_rule.py create mode 100644 test_support/microcosm_build/transport_kernels.py diff --git a/changelog.d/931-transport-helper-bindings.changed.md b/changelog.d/931-transport-helper-bindings.changed.md new file mode 100644 index 0000000000..c62451e354 --- /dev/null +++ b/changelog.d/931-transport-helper-bindings.changed.md @@ -0,0 +1,11 @@ +Every transport kernel now hashes each first-party module its own code, or an object the transport registry hands it, computes through, under the derived binding rule the UK stages and kernels are held to (#931). `source_hash` hashes only the files it is given, so a transport node that called a function in a module its hash did not name replayed a stale cached output when that module changed. Only node keys move. + +- **The rule, shared.** The rule's walk moved from `test_uk_graph` to `test_support/microcosm_build/derived_binding_rule.py`. Each country passes its tables as a declaration there, and the UK's walks are unchanged. The rule's refusal check now counts a store into a container the function itself creates as local. +- **The transport's tables.** `test_transport_binding_rule` holds the 45 kernels `build_transport_registry` registers to the rule: the transport package's 37, the shared atomic geography's four, the ordered solve and the three rules kernels. The transport has its own tables, each entry with a reason a test checks: the eight core modules, 19 core definitions, two refusal-only checks (`graph_inputs.require_outputs` and `reference_profiles._validate_interval_resolution`) and one stop, `target_kernels._COMPILE_MODULES`, which only implementation hashes read. The rule also walks what the registry hands the kernels: the `AxiomEngine` behind the rules kernels and the input closure, and the gate registry behind the battery, whose key also binds its bindings' source closure. +- **What is bound.** 79 bindings in 36 kernels: + - the canonical JSON encoder in 32 kernels. It joins the compile list (`transport/compile_modules.py`) for the seven kernels that hash it, and `_GRAPH_RUNTIME` now excludes only the kernel protocol and the declarations; + - the target surface's factor evidence and its Chronicle feed identity (`reference_factors`, `ledger_artifact`, `chronicle_epoch`), in the kernels that decode or check a surface; + - `artifact_types`, `calibrate.target`, `graph_atomic_geography` and the Axiom input surface where kernels name them; the gate report records (`country_spec`, `gate_battery`, `gates`) in the export and package kernels; `build.trace` in the export readback; `takeup_kernels` in the battery; `frame.rules` and `frame.materialize` in the input closure; `calibrate.hierarchy` and `monetary_binding` where targets or matrices are rebuilt; `frame.exact_money` in the counterfactual bridge, as in the zero solve; + - the ordered solve's internals (exact-k, the L0 gates and their initialization record, the hierarchy, the monetary binding and the encoder) in `calibrate.ordered_adam@1`'s own half, so `calibrate.adam@1`'s hash, which UK kernels and the parity pins embed, does not move. +- **The import record.** `test_transport_compile_modules` drops microcosm#1497's record of unclassified imports, its kernel groups and the record check. It keeps the compile list's closure checks and the geography-label re-key test, and now checks that no transport kernel hashes the compile list. The import scan's record helpers, which nothing else used, go. +- **What moves.** Node keys only. 40 of the 45 transport kernel hashes move; the four atomic geography kernels and the person location keep theirs. All 357 nodes of New Zealand's packaged graph re-key: 353 directly, and the atomic geography and person location nodes by chaining. Belgium composes no graph. No UK kernel hash moves (0 of 99), nor do `calibrate.adam@1`'s or `simulate.rules@1`'s, and nothing of the US's. No value, declaration or generated file moves. diff --git a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py index fa22aadea7..49d60f44ff 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py @@ -50,6 +50,7 @@ from microcosm.frame import concepts as concepts_module from microcosm.frame import input_closure as closure_module from microcosm.frame import unit_construction as units_module +from microcosm.frame.adapters import axiom_input_surface as input_surface_module from microcosm.frame.concept_mapping import ConceptMapping from microcosm.frame.concepts import derive_take_up_draws from microcosm.frame.input_closure import ( @@ -78,9 +79,10 @@ SeedSource, source_hash, ) +from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token -from . import graph_inputs +from . import artifact_types, graph_inputs from . import receipt_contract as receipt_contract_module from .artifact_types import TARGET_SURFACE_TYPE from .graph_inputs import ( @@ -330,6 +332,9 @@ def implementation_hash(self) -> str: return source_hash( type(self), graph_inputs, + # canonical_document_param checks a document parameter's spelling + # through the encoder (microcosm#931). + canonical_json, *self._modules, dependencies=self.capabilities.dependencies, ) @@ -417,7 +422,9 @@ class ConceptsEncodeKernel(_ColumnKernel): """Encode person and household bindings plus declared closure defaults.""" ref = "concepts.encode@1" - _modules = (mapping_module, concepts_module, closure_module) + # InputClosure computes through the Axiom input surface's records + # (microcosm#931). + _modules = (mapping_module, concepts_module, closure_module, input_surface_module) def run(self, context: KernelContext) -> KernelResult: _refuse_take_up(context, self.ref) @@ -526,7 +533,15 @@ class ConceptsEncodeGroupsKernel(_ColumnKernel): """Group bindings execute through G4's encoder and built memberships.""" ref = "concepts.encode_groups@1" - _modules = (mapping_module, units_module, concepts_module, closure_module) + # InputClosure computes through the Axiom input surface's records + # (microcosm#931). + _modules = ( + mapping_module, + units_module, + concepts_module, + closure_module, + input_surface_module, + ) def run(self, context: KernelContext) -> KernelResult: _refuse_take_up(context, self.ref) @@ -784,6 +799,17 @@ def implementation_hash(self) -> str: hierarchy_module, calibration_predicates_module, receipt_contract_module, + # The kernel checks the surface's type (artifact_types), and + # decode_target_surface checks its factor evidence and Chronicle + # feed identity through the next three, rebuilds its registry's + # targets through calibrate.target and decodes its canonical JSON + # (microcosm#931). + artifact_types, + target_kernels.reference_factors_module, + target_kernels.ledger_artifact_module, + target_kernels.chronicle_epoch_module, + target_kernels.target_module, + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py index 58c0437ae5..1e375b5c4b 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py @@ -16,10 +16,15 @@ ``EXCLUDED_DIRECT_IMPORTS`` names every first-party (``microcosm.*``) module a listed module imports but the tuple leaves out, with the reason. A test pins that the two together cover every direct import of every listed module, -so a new import must be listed or excluded. The exclusions cover only the -listed modules' own imports; a second test walks every module each compile -kernel hashes, its own module included, and records the imports no review -has classified yet (microcosm#1497). +so a new import must be listed or excluded. + +The exclusions cover only the listed modules' own imports, and keep a module +off the list, not out of a kernel's hash. Each transport kernel also hashes, +beside this tuple, every first-party module its own code computes through: +the derived binding rule in test_transport_binding_rule holds each kernel's +hash to that, in place of the per-kernel import record microcosm#1497 kept +(microcosm#931). This module is not hashed, so an edit to a reason re-keys +nothing; an edit to the tuple re-keys the kernels. """ from __future__ import annotations @@ -42,6 +47,7 @@ "microcosm.calibrate.hierarchy", "microcosm.calibrate.geography_constants", "microcosm.build.transport.reference_selection", + "microcosm.graph.canonical", ) _COUNTRY_SPEC_LOADER = ( @@ -51,12 +57,16 @@ "validators do not use" ) _GRAPH_RUNTIME = ( - "the graph runtime the listed modules read a node's context, declare " - "artifact types and encode documents with: the kernel protocol, the " - "declarations and the canonical JSON encoder node keys are computed " - "with. The compile kernels hash none of the three; other kernels hash " - "the encoder, among them each UK graph kernel whose code computes " - "through it (microcosm#931)" + "the graph runtime the listed modules read a node's context and declare " + "artifact types with: the kernel protocol and the declarations. Both are " + "core modules every node runs through, which the derived binding rule " + "lists rather than binds (test_transport_binding_rule): node_key hashes " + "each node's declaration, its artifact types included, and every input " + "and output passes through microcosm.graph.kernel's records, so binding " + "either would re-key every node on any edit to it, and the transport " + "graph tests, which run composed graphs through run_graph, catch an edit " + "that changes a value. The canonical JSON encoder, which the listed " + "modules encode and digest documents with, is listed (microcosm#931)" ) EXCLUDED_DIRECT_IMPORTS: Mapping[str, str] = MappingProxyType( @@ -94,7 +104,6 @@ "graph_inputs.context_frame's entity schema; target compilation " "reads no population" ), - "microcosm.graph.canonical": _GRAPH_RUNTIME, "microcosm.graph.decl": _GRAPH_RUNTIME, "microcosm.graph.kernel": _GRAPH_RUNTIME, } diff --git a/packages/microcosm-build/src/microcosm/build/transport/evidence_contract.py b/packages/microcosm-build/src/microcosm/build/transport/evidence_contract.py index 49127385dc..8cb9c42a44 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/evidence_contract.py +++ b/packages/microcosm-build/src/microcosm/build/transport/evidence_contract.py @@ -168,6 +168,12 @@ "microcosm.frame.adapters.axiom_input_surface", "microcosm.frame.adapters.axiom", "microcosm.frame.schema", + # The kernel types its engines by frame.rules' RulesEngine, and + # reads an AxiomEngine's compiled programs (engine_root_inputs), + # whose class builds its tables through frame.materialize + # (microcosm#931). + "microcosm.frame.rules", + "microcosm.frame.materialize", _CANONICAL, _LEAF, ), @@ -179,6 +185,9 @@ "microcosm.calibrate.registry", "microcosm.calibrate.solve", "microcosm.calibrate.target", + # The decoded problem's targets carry their hierarchy + # (microcosm#931). + "microcosm.calibrate.hierarchy", "microcosm.frame.schema", "microcosm.frame.weights", _CANONICAL, diff --git a/packages/microcosm-build/src/microcosm/build/transport/gate_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/gate_kernels.py index 212a027900..eaef447907 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/gate_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/gate_kernels.py @@ -136,6 +136,7 @@ binding_identity, evidence_contract, graph_inputs, + takeup_kernels, target_kernels, ) from .artifact_types import ( @@ -795,6 +796,11 @@ def implementation_hash(self) -> str: calibrate_artifacts_module, frame_bundle_module, frame_schema_module, + # The evidence decoders: decode_target_surface checks a surface's + # factor evidence through reference_factors, and + # decode_cell_diagnostics is takeup_kernels' (microcosm#931). + target_kernels.reference_factors_module, + takeup_kernels, dependencies=self.capabilities.dependencies, ) return hashlib.sha256( diff --git a/packages/microcosm-build/src/microcosm/build/transport/geography_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/geography_kernels.py index 79edaa0db4..40c7b7879b 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/geography_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/geography_kernels.py @@ -18,6 +18,7 @@ import numpy as np import microcosm.build.atomic_geography as atomic_geography_module +import microcosm.build.graph_atomic_geography as graph_atomic_geography_module import microcosm.calibrate.hierarchy as hierarchy_module import microcosm.calibrate.registry as registry_module from microcosm.build.atomic_geography import _U53, encode_atomic_support @@ -35,7 +36,7 @@ ) from microcosm.graph.canonical import canonical_json -from . import graph_inputs, target_kernels +from . import artifact_types, graph_inputs, target_kernels from .artifact_types import TARGET_SURFACE_TYPE from .graph_inputs import ( canonical_document_param, @@ -287,6 +288,17 @@ def implementation_hash(self) -> str: registry_module, hierarchy_module, canonical_json, + # The kernel checks the surface's type (artifact_types) and types + # its support (graph_atomic_geography); decode_target_surface + # checks the surface's factor evidence and its Chronicle feed + # identity through the next three and rebuilds its registry's + # targets through calibrate.target (microcosm#931). + artifact_types, + graph_atomic_geography_module, + target_kernels.reference_factors_module, + target_kernels.ledger_artifact_module, + target_kernels.chronicle_epoch_module, + target_kernels.target_module, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/measure_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/measure_kernels.py index f93aac85ee..198e825071 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/measure_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/measure_kernels.py @@ -46,6 +46,7 @@ Numeric, source_hash, ) +from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token from . import graph_inputs @@ -289,6 +290,9 @@ def implementation_hash(self) -> str: type(self), graph_inputs, reference_intervals_module, + # canonical_document_param checks the measure document's spelling + # through the encoder (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/money_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/money_kernels.py index 392d67036e..4a4739ef03 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/money_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/money_kernels.py @@ -66,6 +66,7 @@ SeedSource, source_hash, ) +from microcosm.graph.canonical import canonical_json from . import graph_inputs from .graph_inputs import ( @@ -193,6 +194,10 @@ def implementation_hash(self) -> str: chronicle_epoch_module, concepts_module, exact_money_module, + # reference_factors digests and replays the factor documents + # through the encoder, and canonical_document_param checks each + # document parameter's spelling through it (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -417,6 +422,9 @@ def implementation_hash(self) -> str: type(self), graph_inputs, exact_money_module, + # canonical_document_param checks the money document's spelling + # through the encoder (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -510,6 +518,9 @@ def implementation_hash(self) -> str: type(self), graph_inputs, exact_money_module, + # canonical_document_param checks the money document's spelling + # through the encoder (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/population_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/population_kernels.py index f8d24121fd..53af1d9729 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/population_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/population_kernels.py @@ -61,6 +61,7 @@ StructuralDelta, source_hash, ) +from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token from . import graph_inputs, target_kernels, unit_declarations @@ -375,6 +376,9 @@ def implementation_hash(self) -> str: transport_module, concepts_module, scaling_module, + # canonical_document_param checks the currency document's + # spelling through the encoder (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py index 46d1431b9e..9616401e6f 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py @@ -30,7 +30,7 @@ from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token -from . import graph_inputs +from . import artifact_types, graph_inputs from .artifact_types import INCOME_CELLS_TYPE from .graph_inputs import ( canonical_document_param, @@ -185,6 +185,12 @@ def implementation_hash(self) -> str: sys.modules[__name__], reporting, graph_inputs, + # The shared contract check names the income cells' type + # (artifact_types); the classifier document's spelling is checked + # through the encoder, which the income deciles also encode their + # cells with (microcosm#931). + artifact_types, + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/takeup_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/takeup_kernels.py index b1d110a293..147327bc46 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/takeup_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/takeup_kernels.py @@ -43,7 +43,7 @@ ) from microcosm.graph.canonical import canonical_json -from . import graph_inputs, target_kernels, weight_quality +from . import artifact_types, graph_inputs, target_kernels, weight_quality from .artifact_types import CELL_DIAGNOSTICS_TYPE from .graph_inputs import ( canonical_document_param, @@ -882,7 +882,13 @@ class TakeupBandsKernel(KernelBase): def implementation_hash(self) -> str: return source_hash( - type(self), graph_inputs, dependencies=self.capabilities.dependencies + type(self), + graph_inputs, + # The kernel decodes its scenario tables and encodes its bands as + # canonical JSON, and checks its document parameters' spelling + # through the encoder (microcosm#931). + canonical_json, + dependencies=self.capabilities.dependencies, ) def run(self, context: KernelContext) -> KernelResult: @@ -1001,7 +1007,16 @@ class TakeupCellDiagnosticsKernel(KernelBase): ) def implementation_hash(self) -> str: - return source_hash(type(self), decode_cell_diagnostics, graph_inputs) + return source_hash( + type(self), + decode_cell_diagnostics, + graph_inputs, + # The kernel types its output (artifact_types), and decodes its + # scenario tables and encodes its output as canonical JSON + # (microcosm#931). + artifact_types, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: require_params(context, self.ref, required=frozenset({"artifact_scenarios"})) diff --git a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py index cdea94844e..002618f5c3 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py @@ -50,6 +50,7 @@ import microcosm.build.calibration_columns as calibration_columns_module import microcosm.build.calibration_predicates as calibration_predicates_module +import microcosm.build.chronicle_epoch as chronicle_epoch_module import microcosm.build.ledger_artifact as ledger_artifact_module import microcosm.build.reference_factors as reference_factors_module import microcosm.build.reference_intervals as reference_intervals_module @@ -61,6 +62,7 @@ import microcosm.calibrate.registry as registry_module import microcosm.calibrate.target as target_module import microcosm.frame.bundle as frame_bundle_module +import microcosm.frame.concepts as frame_concepts_module import microcosm.frame.schema as frame_schema_module import microcosm.frame.unit_construction as units_module import microcosm.frame.weights as frame_weights_module @@ -601,6 +603,12 @@ def implementation_hash(self) -> str: reference_intervals_module, reference_factors_module, reference_profiles_module, + # validate_reference_resolution checks the factor evidence's + # Chronicle feed identity through these two, and the bindings and + # columns are digested as canonical JSON (microcosm#931). + ledger_artifact_module, + chronicle_epoch_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -963,6 +971,15 @@ def implementation_hash(self) -> str: reference_profiles_module, reference_selection_module, *_MATRIX_MODULES, + # decode_target_surface checks the surface's factor evidence and + # its Chronicle feed identity through these three; the interval + # diagnostics count source households by frame.concepts' column + # name and are digested as canonical JSON (microcosm#931). + reference_factors_module, + ledger_artifact_module, + chronicle_epoch_module, + frame_concepts_module, + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/terminal_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/terminal_kernels.py index 76b0c51f97..6267781158 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/terminal_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/terminal_kernels.py @@ -43,6 +43,7 @@ import pandas as pd import microcosm.build.artifact_files as artifact_files_module +import microcosm.build.trace as trace_module import microcosm.calibrate._target_loss_attribution as loss_attribution_module import microcosm.calibrate.artifacts as calibrate_artifacts_module import microcosm.calibrate.diagnostics as calibrate_diagnostics_module @@ -231,6 +232,17 @@ def implementation_hash(self) -> str: frame_bundle_module, frame_schema_module, frame_weights_module, + # decode_target_surface checks the surface's factor evidence and + # its Chronicle feed identity through the first three; the + # surface's targets carry their hierarchy, the rebuilt result's + # constraint matrix checks its monetary binding, and the + # diagnostics are encoded as canonical JSON (microcosm#931). + target_kernels.reference_factors_module, + target_kernels.ledger_artifact_module, + target_kernels.chronicle_epoch_module, + target_kernels.hierarchy_module, + target_kernels.monetary_binding_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -559,6 +571,13 @@ def implementation_hash(self) -> str: frame_bundle_module, frame_schema_module, frame_weights_module, + # require_terminal_gate_reports decodes the gate reports into the + # manifest, phase report and result records these three define, + # and the descriptor is digested as canonical JSON (microcosm#931). + gate_kernels.country_spec_module, + gate_kernels.gate_battery_module, + gate_kernels.gates_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -642,6 +661,11 @@ def implementation_hash(self) -> str: artifact_files_module, materialize_module, axiom_adapter_module, + # file_artifact digests the exported file through + # trace.sha256_file, and the report is encoded as canonical JSON + # (microcosm#931). + trace_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -689,6 +713,13 @@ def implementation_hash(self) -> str: graph_inputs, artifact_types, gate_kernels, + # require_terminal_gate_reports decodes the gate reports into the + # manifest, phase report and result records these three define, + # and the receipt is digested as canonical JSON (microcosm#931). + gate_kernels.country_spec_module, + gate_kernels.gate_battery_module, + gate_kernels.gates_module, + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py b/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py index eb5b6a9953..ab81a0b9cc 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_first_party_imports.py @@ -1,18 +1,15 @@ """Controls for the import scan behind the compile kernels' hash checks. ``test_support.microcosm_build.first_party_imports`` reads a module's source -for the first-party modules it imports, and walks a kernel's hashed modules -through them. These tests pin each import form it resolves, that a -submodule resolves the same way before and after it is imported, and how the -walk treats a shared list's exclusions. +for the first-party modules it imports. These tests pin each import form it +resolves, and that a submodule resolves the same way before and after it is +imported. """ from __future__ import annotations -import importlib import sys import uuid -from types import SimpleNamespace import pytest @@ -20,8 +17,6 @@ from test_support.microcosm_build.first_party_imports import ( defining_module, first_party_imports, - recorded_import_mismatches, - unhashed_imports, ) @@ -96,61 +91,3 @@ def test_a_submodule_resolves_the_same_before_it_is_imported(monkeypatch): monkeypatch.delitem(sys.modules, name, raising=False) monkeypatch.delattr(microcosm.calibrate, "target_selection", raising=False) assert defining_module("microcosm.calibrate", "target_selection") == name - - -def test_the_walk_follows_an_exclusion_but_not_its_imports(scratch_package): - """A list's exclusion covers only the listed module's import of it; the - excluded module's own imports must be hashed.""" - leaf = scratch_package("leaf", "VALUE = 1\n") - middle = scratch_package("middle", f"from {leaf} import VALUE\n") - top = scratch_package("top", f"from {middle} import VALUE\n") - excluded = frozenset({middle}) - assert unhashed_imports({top}, listed=(top,), excluded=excluded) == { - f"{middle} -> {leaf}" - } - # From a module outside the list, the exclusion does not apply. - assert unhashed_imports({top}, listed=(), excluded=excluded) == { - f"{top} -> {middle}" - } - assert unhashed_imports({top, middle, leaf}, listed=(), excluded=excluded) == set() - - -class _RecordingKernel: - """A stand-in kernel whose hash names ``modules`` through ``owner``.""" - - def __init__(self, ref, owner, modules): - self.ref, self.owner, self.modules = ref, owner, modules - - def implementation_hash(self): - return self.owner.source_hash(*self.modules) - - -def test_a_group_record_does_not_cover_another_group(scratch_package): - """Removing a module from one group's hash fails the check even when - another group records the same edge (round 2 of #1493).""" - middle = scratch_package("group_middle", "VALUE = 1\n") - top = scratch_package("group_top", f"from {middle} import VALUE\n") - owner = SimpleNamespace(source_hash=lambda *objects, **dependencies: "x") - top_module, middle_module = ( - importlib.import_module(name) for name in (top, middle) - ) - hashes_both = _RecordingKernel("hashes.both@1", owner, (top_module, middle_module)) - hashes_top = _RecordingKernel("hashes.top@1", owner, (top_module,)) - recorded = {f"{top} -> {middle}": ("top only",)} - - def mismatches(groups): - return recorded_import_mismatches( - groups, listed=(), excluded=frozenset(), hash_inputs={}, recorded=recorded - ) - - assert ( - mismatches( - {"both": ((hashes_both, owner),), "top only": ((hashes_top, owner),)} - ) - == {} - ) - # The first group drops the middle module: the edge is recorded for the - # second group alone, so the first group's walk reports it. - assert mismatches( - {"both": ((hashes_top, owner),), "top only": ((hashes_top, owner),)} - ) == {"both: unhashed and not recorded": [f"{top} -> {middle}"]} diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_binding_rule.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_binding_rule.py new file mode 100644 index 0000000000..0462472ddb --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_binding_rule.py @@ -0,0 +1,548 @@ +"""Every transport kernel's hash binds what its code computes through. + +``source_hash`` does not follow imports, so a node's hash must name every +module whose code can change what the node computes (microcosm#931). The +derived binding rule (``test_support.microcosm_build.derived_binding_rule``), +which test_uk_graph holds the UK stages and kernels to, holds here every +kernel ``build_transport_registry`` registers: the shared transport package +Belgium and New Zealand compose their graphs with, the shared atomic +geography, the ordered solve and the rules kernels. The rule follows what a +kernel's own code names into the module that defines it, to a fixed point, +and each module it reaches must be in what the kernel's key hashes, recorded +at its ``source_hash`` calls. + +The transport's own tables say where the rule stops, each entry with a reason +a test below checks: the core modules every node runs through +(``CORE_MODULES``), the core definitions the kernels name +(``KERNEL_CORE_NAMES``), the checks that only refuse +(``REFUSAL_ONLY_HELPERS``), the kernel stops (``KERNEL_STOPS``) and the names +the rule may stop at outside a key (``KERNEL_UNBOUND_NAMES``, none). Beside +what a kernel's code names, the rule walks what the registry hands it +(``HANDED_CLASSES``, ``HANDED_VALUES``) and holds what a key binds beyond its +``source_hash`` calls (``KEYS_BEYOND_SOURCE_HASH``). The transport lists no +declaration modules: its kernels read spec data through canonical document +parameters, which their node keys hash, not through a stage contract, and +compute through country_spec's code (the gates manifest, the reference +validators), which the rule follows. + +The rule replaced microcosm#1497's per-kernel import record for the transport +compile kernels; test_transport_compile_modules keeps the compile list's own +closure check. What the rule cannot see: a call through an object whose class +the code never names and the tables do not list, a name looked up by a +computed string, and data files read by path. The source codecs are keyed by +their versioned names, not their code, as in the UK. +""" + +from __future__ import annotations + +import ast +import functools +from collections.abc import Callable +from pathlib import Path +from types import MappingProxyType + +import microcosm.build.transport as transport_package +from microcosm.build import graph_atomic_geography +from microcosm.build.transport import ( + binding_identity, + evidence_kernels, + gate_kernels, + population_kernels, + registry, + target_kernels, +) +from microcosm.build.transport.gate_bindings import TRANSPORT_GATE_REGISTRY +from microcosm.build.transport.gate_kernels import GateBatteryKernel +from microcosm.build.transport.registry import build_transport_registry +from microcosm.calibrate.ordered_kernels import CalibrateOrderedAdamKernel +from microcosm.frame import rules_kernels +from microcosm.frame.rules_kernels import ( + SimulateCounterfactualKernel, + SimulateRulesByRefKernel, + SimulateSolveZeroKernel, +) +from microcosm.graph import SeedSource +from test_support.microcosm_build.derived_binding_rule import ( + FIRST_PARTY_SCOPE, + GRAPH_RECORD_REASONS, + BindingRule, + RuleTables, + findings, + hashed_modules, + kernel_classes_defined, + module_definitions, +) +from test_support.microcosm_build.transport_composed import ( + ToyRulesEngine, + toy_rules_bindings, + write_rulespec_tree, +) +from test_support.microcosm_build.transport_kernels import transport_kernels +from test_support.microcosm_build.transport_population import UNIT_RULE + +#: What catches an edit to the Frame, whose three modules share it. +_FRAME_REASON = ( + "the kernels compute on frames, and the executor holds every population " + "version in one. No transport key hashes it; the transport graph tests, " + "which run composed graphs through run_graph (test_transport_composed_graph " + "and test_nz_transport_toy_graph among them), catch an edit that changes a " + "value on their fixtures, and the Frame's own tests the rest" +) +#: The core modules every node runs through, which the rule neither follows +#: nor asks a kernel to bind, each with what runs through it and what catches +#: an edit to it that would change values. Binding one would re-key every +#: node on any edit to it. They are the modules the executor runs every node +#: through (test_the_core_modules_are_on_every_nodes_path). +CORE_MODULES: dict[str, str] = { + "microcosm.frame.bundle": f"the Frame: {_FRAME_REASON}", + "microcosm.frame.schema": f"the Frame's entity schema: {_FRAME_REASON}", + "microcosm.frame.weights": ( + f"the Frame's typed weights and mass records: {_FRAME_REASON}" + ), + "microcosm.graph.executor": ( + "runs every node (run_graph): keys it, projects its declared inputs, " + "calls its kernel and applies its result. No transport key hashes it; " + "the transport graph tests run every node of their composed graphs " + "through it" + ), + "microcosm.graph.population": ( + "the executor applies every node's result to its population version " + "through it (patch); as for the executor, the transport graph tests " + "apply every result of their composed graphs through it" + ), + "microcosm.graph.kernel": ( + "every node's inputs and outputs pass through its KernelContext and " + "KernelResult, and every transport kernel's implementation hash is " + "computed through its source_hash, so an edit that changes what that " + "returns changes every key" + ), + "microcosm.graph.keys": ( + "computes every node's key and its artifacts' keys (node_key), so an " + "edit that changes what it returns changes every key; no transport " + "kernel reads the generator the executor seeds from a key (none " + "declares SeedSource.EXECUTOR; checked)" + ), + "microcosm.graph.decl": ( + "node_key hashes every node's normative declaration and resolves its " + "inputs and population version through the compiled graph, as the " + "executor does, so an edit that changes how a node is declared or " + "what it reads changes its key" + ), +} +#: The core modules' definitions the transport kernels' own code names, each +#: with why naming it binds nothing the kernel's key misses +#: (test_the_core_definitions_the_kernels_name_are_listed). The Frame's +#: definitions are the Frame's reason's. The graph's records, enums and +#: constants take the graph's own reasons, which every country shares; the +#: other four rest on the transport's checks. +KERNEL_CORE_NAMES: dict[tuple[str, str], str] = { + **{ + item: GRAPH_RECORD_REASONS[item] + for item in ( + ("microcosm.graph.decl", "ArtifactOutput"), + ("microcosm.graph.decl", "ArtifactType"), + ("microcosm.graph.decl", "GATE_OUTCOMES"), + ("microcosm.graph.decl", "Node"), + ("microcosm.graph.decl", "Ownership"), + ("microcosm.graph.decl", "ROWS_ALL"), + ("microcosm.graph.decl", "Slice"), + ("microcosm.graph.decl", "StructuralDelta"), + ("microcosm.graph.kernel", "ArtifactValue"), + ("microcosm.graph.kernel", "Capabilities"), + ("microcosm.graph.kernel", "Determinism"), + ("microcosm.graph.kernel", "KernelBase"), + ("microcosm.graph.kernel", "KernelRole"), + ("microcosm.graph.kernel", "Numeric"), + ("microcosm.graph.kernel", "SeedSource"), + ) + }, + ("microcosm.graph.population", "dtype_for_token"): ( + "maps a node's declared dtype token to the dtype its owned cells are " + "stored at. The executor stores every node's owned cells at the dtype " + "it returns (patch, checked), so a kernel that casts its cells through " + "it first hands the executor what the executor stores anyway, and an " + "edit to it moves every node's stored dtypes alike: the transport " + "graph tests catch one on the dtypes their composed graphs carry, and " + "graph.population's own tests the rest" + ), + ("microcosm.graph.kernel", "KernelContext"): ( + "the frozen record of what the executor hands a kernel to read, which " + "a kernel names to annotate it; it only checks and freezes the fields " + "the executor fills, and every node's inputs pass through it " + "(graph.kernel's core reason)" + ), + ("microcosm.graph.kernel", "KernelResult"): ( + "the frozen record a kernel returns its outputs in, which holds what " + "the kernel puts in it and computes nothing; the executor validates " + "every field and applies it through graph.population's patch " + "(graph.population's core reason)" + ), + ("microcosm.graph.kernel", "source_hash"): ( + "hashes the source files of the modules it is given. The evidence " + "producers write their own implementation hash, computed through it " + "(evidence_contract.evidence_implementation_hash), into each evidence " + "header, and the battery recomputes it only to refuse a header that " + "claims another; the rules kernels and the second quantile map name " + "it through the kernel classes they follow whole. An edit that " + "changes what it returns changes those hashes and every key alike " + "(graph.kernel's core reason)" + ), +} +#: Checks that read values only to decide whether to refuse, each with why it +#: cannot change what its caller computes. The rule does not follow them; +#: test_the_refusal_only_helpers_return_and_write_nothing checks each. +REFUSAL_ONLY_HELPERS: dict[tuple[str, str], str] = { + ("microcosm.build.transport.graph_inputs", "require_outputs"): ( + "refuses a node whose declared typed outputs differ from its kernel's " + "(artifact_types.KERNEL_OUTPUTS), and returns nothing. The table it " + "reads types the outputs of every kernel, the problem's and the " + "diagnostics' among them, whose modules (calibrate.artifacts, " + "diagnostics.schema) only the kernels that write those outputs " + "compute through" + ), + ("microcosm.build.reference_profiles", "_validate_interval_resolution"): ( + "binds recorded interval scaling to the selected factors, and to the " + "interval document where it is given one, and refuses on any " + "mismatch, returning nothing. Target compilation, which the compile " + "kernels run, gives it no interval document (checked); " + "calibration.prepare@1 and targets.problem@1, which apply interval " + "tables, hash reference_intervals" + ), +} +#: Where the kernel rule stops, each with why a kernel that names the +#: definition need not hash what lies beyond it; +#: test_the_kernel_rule_stops_only_where_its_reasons_hold checks each. +KERNEL_STOPS: dict[tuple[str, str], str] = { + ("microcosm.build.transport.target_kernels", "_COMPILE_MODULES"): ( + "the imported compile list (compile_modules.transport_compile_modules), " + "which only implementation hashes read: every reference to it in the " + "transport package sits inside an implementation_hash (checked). The " + "walk meets it only because the second quantile map's run names the " + "first quantile map, whose class it follows whole, that hash included; " + "the second quantile map hashes the list itself" + ), +} +#: Names a kernel reads from a module its key does not hash, each with why. +#: None: every name the rule stops at lies inside its kernel's key. +KERNEL_UNBOUND_NAMES: dict[tuple[str, str], str] = {} + +#: The class of the engines build_transport_registry builds, one per rules +#: binding, unless the caller injects pure-Python engines. +ENGINE_CLASS = ("microcosm.frame.adapters.axiom", "AxiomEngine") +_ENGINE_REASON = ( + "build_transport_registry builds each binding's engine as an AxiomEngine " + "and hands the engine mapping to the three rules kernels and to the " + "evidence producers' input closure (checked); their code calls the " + "engines' methods, and names the class only to check an engine's kind" +) +#: Objects the registry hands kernels, whose classes their code need not +#: name, by kernel ref: the class, which the walk follows from the methods the +#: kernel's walk names on it (BindingRule.keyed_walk), and why. +HANDED_CLASSES: dict[str, tuple[tuple[str, str], str]] = { + ref: (ENGINE_CLASS, _ENGINE_REASON) + for ref in ( + "simulate.rules_by_ref@1", + "simulate.counterfactual@1", + "simulate.solve_zero@1", + "transport.input_closure@1", + ) +} +#: Definitions the registry hands kernels by value, by kernel ref, which the +#: walk follows from their definitions, and why. +HANDED_VALUES: dict[str, tuple[tuple[str, str], str]] = { + "gates.battery@1": ( + ("microcosm.build.transport.gate_bindings", "TRANSPORT_GATE_REGISTRY"), + "build_transport_registry builds the battery with the transport gate " + "registry (register_gate_kernel; checked), which the battery's code " + "never names: it evaluates each declared gate through the registry's " + "binding", + ), +} + + +def _binding_sources(kernel) -> frozenset[str]: + """The modules whose source the battery's bindings can run, which its + implementation hash digests (binding_identity.describe).""" + + return frozenset( + module + for binding in kernel.registry.values() + for module in binding_identity.describe(binding)[1] + ) + + +#: Modules a kernel's key binds beyond its ``source_hash`` calls, by kernel +#: ref: how to read them from the kernel, and why. +KEYS_BEYOND_SOURCE_HASH: dict[str, tuple[Callable[[object], frozenset[str]], str]] = { + "gates.battery@1": ( + _binding_sources, + "the battery's implementation hash digests the source closure of each " + "binding (binding_identity.describe) beside its code's hash, so an " + "edit to a module a binding can run re-keys it (checked)", + ), +} + +_RULE = BindingRule( + RuleTables( + core_modules=CORE_MODULES, + refusal_only=REFUSAL_ONLY_HELPERS, + stops=KERNEL_STOPS, + ) +) + + +def _keyed_walk(ref: str, kernel): + """One kernel's walk, with what the registry hands it, against what its + key hashes (``BindingRule.keyed_walk``).""" + + handed_class = HANDED_CLASSES.get(ref) + handed_value = HANDED_VALUES.get(ref) + beyond = KEYS_BEYOND_SOURCE_HASH.get(ref) + return _RULE.keyed_walk( + kernel, + handed_classes=() if handed_class is None else (handed_class[0],), + handed_values=() if handed_value is None else (handed_value[0],), + key=() if beyond is None else beyond[0](kernel), + ) + + +@functools.cache +def _transport_rule_walks() -> MappingProxyType: + """What the rule holds each transport kernel to, by ref, one label each.""" + + return MappingProxyType( + { + ref: {ref: _keyed_walk(ref, kernel)} + for ref, kernel in sorted(transport_kernels().items()) + } + ) + + +def _package_sources(package) -> list[tuple[str, Path]]: + """Each module of ``package`` with its file, the package's own included.""" + + return [ + ( + package.__name__ + if path.stem == "__init__" + else f"{package.__name__}.{path.stem}", + path, + ) + for path in sorted(Path(package.__file__).parent.glob("*.py")) + ] + + +def test_the_rule_sees_every_kernel_the_transport_registers(tmp_path) -> None: + """The kernels the rule walks are every kernel class the transport package + defines, all registered, and the eight the registry registers from + elsewhere: the shared atomic geography's four, the ordered solve and the + three rules kernels. The replicated registration registers each ref with + the class build_transport_registry registers it with, so a new kernel + cannot escape the rule, and each table names a registered kernel.""" + kernels = transport_kernels() + classes = {type(kernel) for kernel in kernels.values()} + defined = kernel_classes_defined(_package_sources(transport_package)) + assert defined <= classes, sorted(cls.__name__ for cls in defined - classes) + assert classes - defined == kernel_classes_defined( + [(graph_atomic_geography.__name__, Path(graph_atomic_geography.__file__))] + ) | { + CalibrateOrderedAdamKernel, + SimulateRulesByRefKernel, + SimulateCounterfactualKernel, + SimulateSolveZeroKernel, + } + # The registry itself, built engine-free with the toy package's engines. + rows = write_rulespec_tree(tmp_path) + prepared = build_transport_registry( + toy_rules_bindings(rows), + tmp_path, + unit_rule=UNIT_RULE, + engines_by_binding={row["id"]: ToyRulesEngine(tmp_path, row) for row in rows}, + ) + assert { + ref: type(prepared.kernels.get(ref)) for ref in prepared.kernels.refs() + } == {ref: type(kernel) for ref, kernel in kernels.items()} + for table in (HANDED_CLASSES, HANDED_VALUES, KEYS_BEYOND_SOURCE_HASH): + assert set(table) <= set(kernels), sorted(set(table) - set(kernels)) + + +def test_every_transport_kernel_hashes_each_helper_it_computes_through() -> None: + """Each module holding code a transport kernel's own methods, or what the + registry hands it, can run to compute a value is in what its key really + hashes, every name the rule stops at lies inside that key or is listed, + and the listed stops are met (microcosm#931). + + The rule starts at what the kernel runs (its methods by method resolution + order, without the hash itself, the methods only the hash calls and the + listed refusal-only methods), stops at the core-module, refusal-only and + stop tables, and follows the engines and the gate registry the registry + hands the kernels that take them (``BindingRule.keyed_walk``).""" + found = findings(_transport_rule_walks()) + assert not found.unhashed, found.unhashed + assert found.unlisted == set(KERNEL_UNBOUND_NAMES), sorted( + found.unlisted ^ set(KERNEL_UNBOUND_NAMES) + ) + assert found.met == set(KERNEL_STOPS), sorted(found.met ^ set(KERNEL_STOPS)) + + +def test_the_core_modules_are_on_every_nodes_path() -> None: + """Each listed core module is on every node's path, as its reason says: + the walk from run_graph, which keys, runs and patches every node, meets + each. The key module's reason holds too: no transport kernel reads the + generator the executor seeds from a node key.""" + off_path = _RULE.core_modules_off_run_graph_path() + assert not off_path, off_path + seeded = { + ref + for ref, kernel in transport_kernels().items() + if kernel.capabilities.seed_source is SeedSource.EXECUTOR + } + assert not seeded, sorted(seeded) + + +def test_the_core_definitions_the_kernels_name_are_listed() -> None: + """The definitions of the core modules other than the Frame's that the + transport kernels' code names are exactly ``KERNEL_CORE_NAMES``, and the + executor stores every node's owned cells through the dtype tokens' map, + as that entry's reason says.""" + named = findings(_transport_rule_walks()).core_names + assert named == set(KERNEL_CORE_NAMES), sorted(named ^ set(KERNEL_CORE_NAMES)) + patching = _RULE.helper_walk( + ("microcosm.graph.population", "patch"), follow_core=True + ) + assert "dtype_for_token" in patching.reach["microcosm.graph.population"] + + +def test_the_refusal_only_helpers_return_and_write_nothing() -> None: + """Each listed check returns nothing and writes nothing + (``writes_nothing``), and target compilation gives the interval check no + interval document, as its reason says.""" + violations = _RULE.refusal_only_violations() + assert not violations, violations + compile_surface = module_definitions( + target_kernels.__name__, FIRST_PARTY_SCOPE + ).definitions["compile_target_surface"] + calls = [ + sub + for sub in ast.walk(compile_surface) + if isinstance(sub, ast.Call) + and getattr(sub.func, "id", None) == "validate_reference_resolution" + ] + assert calls + for call in calls: + assert "interval_document" not in {item.arg for item in call.keywords} + + +def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: + """The compile list is met only through the first quantile map's class, + which the second quantile map's run names, and every reference to it in + the transport package sits inside an implementation hash.""" + namers: dict[tuple[str, str], set] = {} + for walks in _transport_rule_walks().values(): + for walk, _ in walks.values(): + for stop, by in walk.met.items(): + namers.setdefault(stop, set()).update(by) + compile_list = (target_kernels.__name__, "_COMPILE_MODULES") + assert namers == { + compile_list: {(population_kernels.__name__, "TransportQuantileMapKernel")} + } + for module, path in _package_sources(transport_package): + tree = ast.parse(path.read_text(encoding="utf-8")) + hashing = { + id(sub) + for function in ast.walk(tree) + if isinstance(function, ast.FunctionDef) + and function.name == "implementation_hash" + for sub in ast.walk(function) + } + for sub in ast.walk(tree): + reads = (isinstance(sub, ast.Name) and sub.id == compile_list[1]) or ( + isinstance(sub, ast.Attribute) and sub.attr == compile_list[1] + ) + if reads and not isinstance(sub.ctx, ast.Store): + assert id(sub) in hashing, (module, sub.lineno) + + +def test_the_registry_hands_the_kernels_what_the_tables_list() -> None: + """build_transport_registry builds AxiomEngine engines and hands their + mapping to exactly the rules kernels and the evidence producers, which + hand it to the input closure alone (it also returns the mapping, in the + TransportRegistry its caller runs the graph with), and builds the battery + with the transport gate registry: the objects ``HANDED_CLASSES`` and + ``HANDED_VALUES`` list for those kernels.""" + builder = module_definitions(registry.__name__, FIRST_PARTY_SCOPE).definitions[ + "build_transport_registry" + ] + + def called(node, module: str) -> list[tuple[tuple[str, str] | None, ast.Call]]: + return [ + (_RULE.resolve(module, sub.func.id), sub) + for sub in ast.walk(node) + if isinstance(sub, ast.Call) and isinstance(sub.func, ast.Name) + ] + + def handed(node, module: str, argument: str) -> set: + return { + callee + for callee, call in called(node, module) + if any( + isinstance(value, ast.Name) and value.id == argument + for value in call.args + ) + } + + calls = called(builder, registry.__name__) + assert ENGINE_CLASS in {callee for callee, _ in calls} + assert handed(builder, registry.__name__, "engine_mapping") == { + (rules_kernels.__name__, "SimulateRulesByRefKernel"), + (rules_kernels.__name__, "SimulateCounterfactualKernel"), + (rules_kernels.__name__, "SimulateSolveZeroKernel"), + (evidence_kernels.__name__, "register_evidence_kernels"), + (registry.__name__, "TransportRegistry"), + } + producers = module_definitions( + evidence_kernels.__name__, FIRST_PARTY_SCOPE + ).definitions["register_evidence_kernels"] + assert handed(producers, evidence_kernels.__name__, "engines") == { + (evidence_kernels.__name__, "TransportInputClosureKernel") + } + kernels = transport_kernels() + assert {type(kernels[ref]) for ref in HANDED_CLASSES} == { + SimulateRulesByRefKernel, + SimulateCounterfactualKernel, + SimulateSolveZeroKernel, + evidence_kernels.TransportInputClosureKernel, + } + (battery,) = [ + call + for callee, call in calls + if callee == (gate_kernels.__name__, "register_gate_kernel") + ] + value = battery.args[1] + assert isinstance(value, ast.Name) + assert ( + _RULE.resolve(registry.__name__, value.id) + == HANDED_VALUES["gates.battery@1"][0] + ) + + +def test_the_battery_key_binds_its_bindings_sources(monkeypatch) -> None: + """The modules the battery's key binds beyond its ``source_hash`` calls + are its bindings' source closures: a changed digest of one of them, which + its code's hash does not cover, re-keys the battery.""" + kernel = transport_kernels()["gates.battery@1"] + sources, _ = KEYS_BEYOND_SOURCE_HASH["gates.battery@1"] + beyond = sorted(sources(kernel) - hashed_modules(kernel)) + assert beyond + before = GateBatteryKernel(TRANSPORT_GATE_REGISTRY).implementation_hash() + describe = binding_identity.describe + + def edited(binding, *, path: str = "binding"): + description, digests = describe(binding, path=path) + return description, { + module: "0" * 64 if module == beyond[0] else digest + for module, digest in digests.items() + } + + monkeypatch.setattr(binding_identity, "describe", edited) + assert GateBatteryKernel(TRANSPORT_GATE_REGISTRY).implementation_hash() != before diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py index 2aa1ba903a..1b9137a15c 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_compile_modules.py @@ -13,11 +13,16 @@ - the shared list's own: every first-party import of a listed module is listed, or excluded with a reason in ``compile_modules``; -- each kernel's: the walk from every module the kernel hashes, its own - module included, meets only modules the kernel hashes, the list's - exclusions, or the imports recorded in ``UNCLASSIFIED_IMPORTS`` for that - kernel's group, which no review has classified yet (microcosm#1497). That - mapping may only shrink. +- each kernel's: the derived binding rule in test_transport_binding_rule + holds what each transport kernel's hash covers to what its code computes + through (``test_every_transport_kernel_hashes_each_helper_it_computes_through``). + It replaced microcosm#1497's per-kernel import record for these kernels + (microcosm#931). + +Beside a pin of an edit that must re-key the compile kernels, this file keeps +what that rule does not cover: the compile list is a module a kernel's file +imports only to build its hash (``HASH_INPUTS``), so it is still imported +where listed, and no transport kernel hashes it. """ from __future__ import annotations @@ -34,14 +39,14 @@ takeup_kernels, target_kernels, ) +from test_support.microcosm_build.derived_binding_rule import hashed_modules from test_support.microcosm_build.first_party_imports import ( HashInput, assert_an_edit_rekeys, first_party_imports, - hashed_module_names, - recorded_import_mismatches, unlisted_imports, ) +from test_support.microcosm_build.transport_kernels import transport_kernels #: Every kernel whose implementation hash includes ``_COMPILE_MODULES``, with #: the module whose ``source_hash`` it calls. The first five compile @@ -58,35 +63,13 @@ ) _KERNEL_IDS = [kernel.ref for kernel, _ in KERNELS_HASHING_THE_COMPILE_MODULES] -#: The kernels grouped by shared hash: the two CREATE versions share one -#: class body, so one hashed set; every other kernel is its own group. -COMPILE = "targets.compile@1" -COMPARE = "takeup.compare@1" -GAP = "takeup.gap@1" -CREATE = "transport.create@1 and @2" -QUANTILE_MAP_1 = "transport.quantile_map@1" -QUANTILE_MAP_2 = "transport.quantile_map@2" -KERNEL_GROUPS = MappingProxyType( - { - COMPILE: ((target_kernels.TARGETS_COMPILE, target_kernels),), - COMPARE: ((target_kernels.TAKEUP_COMPARE, target_kernels),), - GAP: ((takeup_kernels.TAKEUP_GAP, takeup_kernels),), - CREATE: ( - (population_kernels.TRANSPORT_CREATE, population_kernels), - (population_kernels.TRANSPORT_CREATE_DECLARED, population_kernels), - ), - QUANTILE_MAP_1: ( - (population_kernels.TRANSPORT_QUANTILE_MAP, population_kernels), - ), - QUANTILE_MAP_2: ( - (quantile_kernels.TRANSPORT_QUANTILE_MAP_V2, quantile_kernels), - ), - } -) - -#: Imports a kernel's module makes only to build the hash, and the importers -#: the reason covers. They stay unhashed on purpose, so an edit to the list -#: re-keys only the kernels whose hashed modules it adds or removes. +#: Imports a kernel's module makes only to build an implementation hash, and +#: the importers the reason covers. The derived binding rule in +#: test_transport_binding_rule leaves every implementation hash out of what a +#: kernel's code computes through, so such an import binds nothing. They stay +#: unhashed on purpose, so an edit to the list re-keys only the kernels whose +#: hashed modules it adds or removes +#: (test_no_transport_kernel_hashes_a_hash_input). HASH_INPUTS = MappingProxyType( { "microcosm.build.transport.compile_modules": HashInput( @@ -99,478 +82,6 @@ } ) -#: Every ``importer -> imported`` edge the walk from a kernel group's hashed -#: modules meets that the group does not hash and no review has classified, -#: with the groups that leave it unhashed. Recorded as found when the -#: per-kernel check was added; microcosm#1497 classifies them. Hash an entry -#: in the kernels that run it, move what they run into a module they hash, or -#: record why they need not hash it, then delete it here. The check fails on -#: an edge a group meets that is not recorded for it and on an edge recorded -#: for a group that no longer meets it, so the mapping can only shrink, and -#: removing a module from one group's hash fails even where another group -#: records the same edge. -UNCLASSIFIED_IMPORTS = MappingProxyType( - { - "microcosm.build.trace -> microcosm.build": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.build.calibration_columns": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.build.calibration_predicates": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.plan -> microcosm.build.observation": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.build.reference_intervals": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.canonical": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.canonical": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.take_up_semantics -> microcosm.build.spec_engine.canonical": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.engine_abi": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.errors": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.errors": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.schemas -> microcosm.build.spec_engine.errors": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.take_up_semantics -> microcosm.build.spec_engine.errors": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.yaml12 -> microcosm.build.spec_engine.errors": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.canonical -> microcosm.build.spec_engine.model": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.model": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.model": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.resolver": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.schemas": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.schemas": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.model -> microcosm.build.spec_engine.seeds": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.engine_abi -> microcosm.build.spec_engine.take_up_semantics": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.loader -> microcosm.build.spec_engine.yaml12": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.build.transport.column_kernels": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.build.transport.unit_declarations": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.build.transport.unit_declarations": ( - COMPILE, - COMPARE, - GAP, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.engine_abi -> microcosm.build.us_runtime.multispine_pool": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.calibrate.artifacts": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.calibrate.matrix": ( - COMPILE, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.calibrate.artifacts -> microcosm.calibrate.matrix": ( - COMPILE, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.calibrate.monetary_binding": ( - COMPILE, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.calibrate.artifacts -> microcosm.calibrate.solve": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.spec_engine.engine_abi -> microcosm.frame.adapters.policyengine_us": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.adapters.policyengine_us_concepts": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.frame.transport -> microcosm.frame.adapters.policyengine_us_concepts": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.plan -> microcosm.frame.bundle": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.bundle": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.frame.bundle": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.calibrate.artifacts -> microcosm.frame.bundle": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.graph.kernel -> microcosm.frame.bundle": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.concept_mapping": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.reference_intervals -> microcosm.frame.concepts": ( - COMPILE, - COMPARE, - GAP, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.concepts": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.frame.transport -> microcosm.frame.concepts": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.materialize": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.frame.transport -> microcosm.frame.materialize": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.scaling": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.frame.transport -> microcosm.frame.scaling": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.schema": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.frame.schema": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.frame.bundle -> microcosm.frame.schema": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.unit_construction": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.frame.unit_construction": ( - COMPILE, - COMPARE, - GAP, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.frame.weights": ( - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.target_kernels -> microcosm.frame.weights": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.calibrate.artifacts -> microcosm.frame.weights": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.frame.bundle -> microcosm.frame.weights": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.graph.kernel -> microcosm.frame.weights": ( - COMPILE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.reference_intervals -> microcosm.graph.canonical": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.quantile_kernels -> microcosm.graph.canonical": ( - QUANTILE_MAP_2, - ), - "microcosm.build.transport.takeup_kernels -> microcosm.graph.canonical": (GAP,), - "microcosm.build.transport.target_kernels -> microcosm.graph.canonical": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.calibrate.artifacts -> microcosm.graph.canonical": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.graph.decl": ( - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.takeup_kernels -> microcosm.graph.decl": (GAP,), - "microcosm.build.transport.target_kernels -> microcosm.graph.decl": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.calibrate.artifacts -> microcosm.graph.decl": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.graph.canonical -> microcosm.graph.decl": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.graph.kernel -> microcosm.graph.decl": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.graph.kernel": ( - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.quantile_kernels -> microcosm.graph.kernel": ( - QUANTILE_MAP_2, - ), - "microcosm.build.transport.takeup_kernels -> microcosm.graph.kernel": (GAP,), - "microcosm.build.transport.target_kernels -> microcosm.graph.kernel": ( - COMPILE, - COMPARE, - GAP, - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - "microcosm.build.transport.population_kernels -> microcosm.graph.population": ( - CREATE, - QUANTILE_MAP_1, - QUANTILE_MAP_2, - ), - } -) - def _compile_list() -> ModuleType: """The shared list and its exclusions (``compile_modules``, microcosm#1492).""" @@ -588,13 +99,36 @@ def test_an_edit_to_the_geography_labels_rekeys_the_kernel( assert_an_edit_rekeys(kernel, geography_constants_module, tmp_path, monkeypatch) -def test_the_compile_list_lives_in_a_module_no_compile_kernel_hashes(): - """target_kernels is hashed by 13 kernels; the list is not, so an edit to - a name or a reason no longer re-keys all of them.""" +def test_no_transport_kernel_hashes_a_hash_input(): + """What the derived binding rule does not cover (microcosm#931). + + The rule in test_transport_binding_rule holds each transport kernel's + hash to what its code computes through, in place of microcosm#1497's + per-kernel import record, and leaves every implementation hash out of + that code, so an import made only to build a hash binds nothing; its + stop at ``target_kernels._COMPILE_MODULES`` rests on only hashes reading + the list. It does not check the other side, that such a module stays + out of the hashes. Each hash input is still imported by every importer + listed for it, the compile kernels hash the list's modules, and no + transport kernel, of every kernel the registry registers, hashes a hash + input: many kernels hash target_kernels and none the list, so an edit to + a name or a reason in it re-keys no kernel, and an edit to the tuple + re-keys only the kernels whose hashed modules it adds or removes.""" compile_list = _compile_list() assert target_kernels._COMPILE_MODULES == compile_list.transport_compile_modules() - for kernel, module in KERNELS_HASHING_THE_COMPILE_MODULES: - assert compile_list.__name__ not in hashed_module_names(kernel, module) + for name, hash_input in HASH_INPUTS.items(): + for importer in hash_input.importers: + assert name in first_party_imports(importer), (importer, name) + hashed = { + ref: hashed_modules(kernel) for ref, kernel in transport_kernels().items() + } + assert {kernel.ref for kernel, _ in KERNELS_HASHING_THE_COMPILE_MODULES} == { + ref + for ref, modules in hashed.items() + if set(compile_list.TRANSPORT_COMPILE_MODULE_NAMES) <= modules + } + holding = {ref for ref, modules in hashed.items() if set(HASH_INPUTS) & modules} + assert not holding, sorted(holding) def test_every_direct_import_of_a_compile_module_is_listed_or_excluded(): @@ -613,21 +147,3 @@ def test_compile_exclusions_are_live_and_never_listed(): imported = set().union(*map(first_party_imports, listed)) assert excluded <= imported, sorted(excluded - imported) assert not excluded & set(listed) - - -def test_every_import_a_compile_kernel_reaches_is_hashed_or_recorded(): - """The walk covers each kernel's own module and every module hashed - beside the shared list, not only the list (microcosm#1492 review), and - the record is kept per group of kernels sharing one hash.""" - compile_list = _compile_list() - assert {kernel for group in KERNEL_GROUPS.values() for kernel, _ in group} == { - kernel for kernel, _ in KERNELS_HASHING_THE_COMPILE_MODULES - } - mismatches = recorded_import_mismatches( - KERNEL_GROUPS, - listed=compile_list.TRANSPORT_COMPILE_MODULE_NAMES, - excluded=frozenset(compile_list.EXCLUDED_DIRECT_IMPORTS), - hash_inputs=HASH_INPUTS, - recorded=UNCLASSIFIED_IMPORTS, - ) - assert not mismatches, mismatches diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_evidence_kernels.py index 6ca5b97ae8..3ea2dc0935 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_evidence_kernels.py @@ -460,6 +460,10 @@ def test_the_lists_are_pinned(self) -> None: "microcosm.frame.adapters.axiom", # S2: the producer calls schema.entity_id_column. "microcosm.frame.schema", + # microcosm#931: the engines are RulesEngines, and the + # AxiomEngine class builds its tables through materialize. + "microcosm.frame.rules", + "microcosm.frame.materialize", "microcosm.graph.canonical", "microcosm.build.transport.evidence_contract", ), @@ -472,6 +476,8 @@ def test_the_lists_are_pinned(self) -> None: "microcosm.calibrate.solve", # S2: the trace finds these in run(); the list missed them. "microcosm.calibrate.target", + # microcosm#931: the decoded problem's targets carry it. + "microcosm.calibrate.hierarchy", "microcosm.frame.schema", "microcosm.frame.weights", "microcosm.graph.canonical", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 8f5fadea70..84d87df4a3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -11,6 +11,7 @@ from microcosm.graph.canonical import canonical_json from test_support.microcosm_build.derived_binding_rule import ( FIRST_PARTY_SCOPE, + GRAPH_RECORD_REASONS, BindingRule, KernelWalk, KeyedWalk, @@ -1102,7 +1103,11 @@ def changed_module_bytes(path): #: key misses (test_the_core_definitions_the_kernels_name_are_listed). The #: Frame's definitions, which kernels build and read frames with, are the #: Frame's reason's; a stage's transform meets the Frame's modules alone -#: (test_the_core_modules_are_on_every_nodes_path). +#: (test_the_core_modules_are_on_every_nodes_path). The graph's records and +#: enums take their reasons from the graph's own facts +#: (derived_binding_rule.GRAPH_RECORD_REASONS), which every country shares; the +#: kernels name some of them through microcosm.graph's lazy exports, which the +#: walk follows since microcosm#931 ruling 8. _KERNEL_CORE_NAMES: dict[tuple[str, str], str] = { ("microcosm.graph.population", "dtype_for_token"): ( "maps a node's declared dtype token to the dtype its owned cells are " @@ -1113,53 +1118,19 @@ def changed_module_bytes(path): "the dtypes its spine fixture carries, and graph.population's own " "tests the rest" ), - ("microcosm.graph.decl", "ArtifactType"): ( - "a frozen record of an artifact type's name and version, which a " - "kernel names to declare or check its artifacts' types; node_key " - "hashes each node's declared artifact types as part of its normative " - "declaration" - ), - ("microcosm.graph.decl", "ArtifactOutput"): ( - "a frozen record pairing an artifact's name with its type, which the " - "atomic geography gate names to declare and check its outputs; " - "node_key hashes each node's declared artifact outputs" - ), - ("microcosm.graph.decl", "StructuralDelta"): ( - "the enum a kernel's capabilities declare its row-set change in; the " - "executor refuses a kernel whose declared change differs from its " - "node's, and node_key binds the declared value with the rest of the " - "kernel's capabilities" - ), - ("microcosm.graph.kernel", "KernelRole"): ( - "the enum a kernel's capabilities declare its role in, by which the " - "executor holds a gate's outcome to the gate outcomes and derives a " - "release's tier from its gate ancestors; node_key binds the declared " - "value with the rest of the kernel's capabilities" - ), - # The kernels name the entries below through microcosm.graph's lazy - # exports, which the walk follows since microcosm#931 ruling 8. - ("microcosm.graph.kernel", "Capabilities"): ( - "the frozen record of a kernel's declared contract, which a kernel " - "names to declare its capabilities; it only checks the types and " - "consistency of its fields, and node_key binds every field it holds " - "(keys._capabilities_projection)" - ), - ("microcosm.graph.kernel", "Determinism"): ( - "the enum a kernel's capabilities declare its determinism in, whether " - "its output is a function of its inputs and seed; node_key binds the " - "declared value with the rest of the kernel's capabilities" - ), - ("microcosm.graph.kernel", "Numeric"): ( - "the enum a kernel's capabilities declare its numbers' " - "reproducibility in, from which the executor resolves the numeric " - "scope of each column a reader is given; node_key binds the declared " - "value with the rest of the kernel's capabilities" - ), - ("microcosm.graph.kernel", "SeedSource"): ( - "the enum a kernel's capabilities declare its seed source in, where a " - "seeded kernel takes its randomness from; node_key binds the declared " - "value with the rest of the kernel's capabilities" - ), + **{ + item: GRAPH_RECORD_REASONS[item] + for item in ( + ("microcosm.graph.decl", "ArtifactType"), + ("microcosm.graph.decl", "ArtifactOutput"), + ("microcosm.graph.decl", "StructuralDelta"), + ("microcosm.graph.kernel", "KernelRole"), + ("microcosm.graph.kernel", "Capabilities"), + ("microcosm.graph.kernel", "Determinism"), + ("microcosm.graph.kernel", "Numeric"), + ("microcosm.graph.kernel", "SeedSource"), + ) + }, ("microcosm.graph.kernel", "KernelContext"): ( "the frozen record of what the executor hands a kernel to read, which " "a kernel names to annotate it; it only checks and freezes the fields " diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/ordered_kernels.py b/packages/microcosm-calibrate/src/microcosm/calibrate/ordered_kernels.py index 9149042eed..8aa762dab1 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/ordered_kernels.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/ordered_kernels.py @@ -52,7 +52,12 @@ import numpy as np import microcosm.calibrate.artifacts as artifacts_module +import microcosm.calibrate.exact_k as exact_k_module +import microcosm.calibrate.gates as l0_gates_module +import microcosm.calibrate.hierarchy as hierarchy_module +import microcosm.calibrate.initialization as initialization_module import microcosm.calibrate.kernels as adam_kernels_module +import microcosm.calibrate.monetary_binding as monetary_binding_module from microcosm.calibrate import calibrate from microcosm.calibrate.artifacts import ( PROBLEM_TYPE, @@ -154,7 +159,10 @@ def implementation_hash(self) -> str: diagnostics, the matrix compiler, the target model and the frame modules; this kernel runs the same solve, so any edit there re-keys both. The adapter half covers this module and the artifact codec it - decodes and encodes through. + decodes and encodes through, and what the solve computes through + beyond the modules adam@1 binds (microcosm#931). Those go here, not in + adam@1's own hash, which the UK graph kernels and the parity pins + embed. """ return hashlib.sha256( @@ -165,6 +173,18 @@ def implementation_hash(self) -> str: type(self), artifacts_module, adam_kernels_module, + # What the solve computes through beyond adam@1's + # modules: the exact-k feasibility check, the L0 + # gates and their initialization record, the + # targets' hierarchy and the matrix's monetary binding + # check; the codec packs the artifacts as canonical + # JSON (microcosm#931). + exact_k_module, + l0_gates_module, + initialization_module, + hierarchy_module, + monetary_binding_module, + canonical_json, dependencies=self.capabilities.dependencies, ), } diff --git a/packages/microcosm-frame/src/microcosm/frame/rules_kernels.py b/packages/microcosm-frame/src/microcosm/frame/rules_kernels.py index 7d3b9b4030..3da19204f4 100644 --- a/packages/microcosm-frame/src/microcosm/frame/rules_kernels.py +++ b/packages/microcosm-frame/src/microcosm/frame/rules_kernels.py @@ -50,6 +50,7 @@ Ownership, source_hash, ) +from microcosm.graph.canonical import canonical_json __all__ = [ "SimulateRulesByRefKernel", @@ -77,12 +78,12 @@ class SimulateRulesByRefKernel(KernelBase): outputs check, its all-rows rule, and its receipt, unchanged. The implementation hash binds this module, ``microcosm.frame.kernels``, - the frame modules ``simulate.rules@1`` binds, and the source of each - distinct adapter *class*. It does not bind the references: a second - binding of a class already bound leaves every node key where it was. A - binding of a new adapter class adds that class's source to the hash and so - re-keys every node of this kernel, as replacing an adapter class does for - ``simulate.rules@1``. + the frame modules ``simulate.rules@1`` binds, the canonical JSON encoder + and the source of each distinct adapter *class*. It does not bind the + references: a second binding of a class already bound leaves every node + key where it was. A binding of a new adapter class adds that class's + source to the hash and so re-keys every node of this kernel, as replacing + an adapter class does for ``simulate.rules@1``. Raises: TypeError: If ``engines`` is not a mapping, or a binding is not a @@ -124,7 +125,12 @@ def engine_refs(self) -> tuple[str, ...]: return tuple(self._delegates) def implementation_hash(self) -> str: - """Hash the routing and delegate kernels plus every bound adapter class.""" + """Hash the routing and delegate kernels plus every bound adapter class. + + The encoder is bound too: an AxiomEngine builds its views as + AxiomView records, which compare and hash by their canonical JSON + (microcosm#931). + """ return source_hash( type(self), @@ -133,6 +139,7 @@ def implementation_hash(self) -> str: frame_bundle_module, frame_rules_module, frame_schema_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -235,6 +242,26 @@ def __init__( ), ) + def implementation_hash(self) -> str: + """Bind the exact helpers as imports are not followed by source_hash. + + ``_prepare`` and ``_evaluate`` name frame.exact_money's helpers in + their quantized branches, which the zero solve runs; the bridge, + which shares them, binds them too, with the encoder the by-ref + kernel binds (microcosm#931). + """ + return source_hash( + type(self), + SimulateRulesKernel, + *self._adapter_classes, + frame_bundle_module, + frame_rules_module, + frame_schema_module, + frame_exact_money_module, + canonical_json, + dependencies=self.capabilities.dependencies, + ) + def run(self, context: KernelContext) -> KernelResult: components, terms, frame = self._prepare(context) values = self._evaluate(frame, components, terms) @@ -736,19 +763,6 @@ class SimulateSolveZeroKernel(SimulateCounterfactualKernel): ref = "simulate.solve_zero@1" - def implementation_hash(self) -> str: - """Bind the exact helpers as imports are not followed by source_hash.""" - return source_hash( - type(self), - SimulateRulesKernel, - *self._adapter_classes, - frame_bundle_module, - frame_rules_module, - frame_schema_module, - frame_exact_money_module, - dependencies=self.capabilities.dependencies, - ) - def run(self, context: KernelContext) -> KernelResult: components, terms, frame = self._prepare(context, solver=True) output_keys = {(term["entity"], term["column"]) for term in terms} diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_rules_kernels.py b/packages/microcosm-frame/tests/engine_free/shared/test_rules_kernels.py index 3bc855ac3b..389003ddb9 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_rules_kernels.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_rules_kernels.py @@ -24,6 +24,7 @@ import microcosm.frame.bundle as frame_bundle_module import microcosm.frame.rules as frame_rules_module import microcosm.frame.schema as frame_schema_module +import microcosm.graph.canonical as graph_canonical_module from microcosm.frame import ( EntitySchema, ExportContract, @@ -219,6 +220,7 @@ def _hash_of(classes) -> str: frame_bundle_module, frame_rules_module, frame_schema_module, + graph_canonical_module, ) @@ -409,6 +411,7 @@ def test_implementation_hash_binds_both_kernels_and_each_adapter_class( frame_bundle_module, frame_rules_module, frame_schema_module, + graph_canonical_module, ) assert ( kernel.implementation_hash() @@ -456,6 +459,7 @@ def test_the_hash_is_independent_of_binding_order(self) -> None: frame_bundle_module, frame_rules_module, frame_schema_module, + graph_canonical_module, ) orders = list(itertools.permutations(bindings)) assert len(orders) == 6 diff --git a/test_support/microcosm_build/derived_binding_rule.py b/test_support/microcosm_build/derived_binding_rule.py index 60a2611f62..f5e2c3f1d1 100644 --- a/test_support/microcosm_build/derived_binding_rule.py +++ b/test_support/microcosm_build/derived_binding_rule.py @@ -104,6 +104,99 @@ #: The builtins whose call returns a new container (``local_containers``). FRESH_CONTAINER_CALLS = frozenset({"dict", "list", "set"}) +#: The graph's records, enums and constants that kernels' code names, each +#: with why naming it binds nothing a kernel's key misses. They are facts +#: about the graph, the same for every country, so a country's table of the +#: core definitions its kernels name takes these reasons from here and gives +#: its own only to definitions whose reason rests on its own checks. +GRAPH_RECORD_REASONS: Mapping[tuple[str, str], str] = MappingProxyType( + { + ("microcosm.graph.decl", "ArtifactType"): ( + "a frozen record of an artifact type's name and version, which a " + "kernel names to declare or check its artifacts' types; node_key " + "hashes each node's declared artifact types as part of its " + "normative declaration" + ), + ("microcosm.graph.decl", "ArtifactOutput"): ( + "a frozen record pairing an artifact's name with its type, which a " + "kernel names to declare and check its outputs, as the shared atomic " + "geography gate does; node_key hashes each node's declared artifact " + "outputs" + ), + ("microcosm.graph.decl", "StructuralDelta"): ( + "the enum a kernel's capabilities declare its row-set change in; the " + "executor refuses a kernel whose declared change differs from its " + "node's, and node_key binds the declared value with the rest of the " + "kernel's capabilities" + ), + ("microcosm.graph.decl", "GATE_OUTCOMES"): ( + "the closed set of outcomes a gate may record, to which the executor " + "holds every gate node's outcome; a kernel names it to refuse a gate " + "report recording another, and an edit to it changes what every gate " + "may record alike" + ), + ("microcosm.graph.decl", "Node"): ( + "the frozen record of a node's declaration, which a kernel names to " + "annotate or check the node its context carries; node_key hashes " + "each node's normative declaration" + ), + ("microcosm.graph.decl", "Ownership"): ( + "the enum of what a node promises about the cells it owns, which a " + "kernel names to check its node's declared outputs; node_key hashes " + "each node's declared outputs with their ownership" + ), + ("microcosm.graph.decl", "ROWS_ALL"): ( + "the rows token of a slice or output that covers every row, which a " + "kernel names to refuse a node that declares fewer; node_key hashes " + "each node's slices and outputs with their rows" + ), + ("microcosm.graph.decl", "Slice"): ( + "the frozen record of what a node reads, one entity's columns under " + "a row mask, which a kernel names to annotate or check its node's " + "slices; it only checks its fields, and node_key hashes each node's " + "declared slices" + ), + ("microcosm.graph.kernel", "ArtifactValue"): ( + "the frozen record of an artifact's verified bytes and typed " + "provenance that the executor hands a kernel, which a kernel names " + "to annotate its inputs; it only checks and freezes its fields, and " + "node_key binds each typed artifact input's producer key and type" + ), + ("microcosm.graph.kernel", "KernelBase"): ( + "the base class of the kernels, whose default implementation hash " + "hashes a kernel class's own module; a kernel's code names it as a " + "class's base, and it computes nothing a node's values depend on" + ), + ("microcosm.graph.kernel", "KernelRole"): ( + "the enum a kernel's capabilities declare its role in, by which the " + "executor holds a gate's outcome to the gate outcomes and derives a " + "release's tier from its gate ancestors; node_key binds the declared " + "value with the rest of the kernel's capabilities" + ), + ("microcosm.graph.kernel", "Capabilities"): ( + "the frozen record of a kernel's declared contract, which a kernel " + "names to declare its capabilities; it only checks the types and " + "consistency of its fields, and node_key binds every field it holds " + "(keys._capabilities_projection)" + ), + ("microcosm.graph.kernel", "Determinism"): ( + "the enum a kernel's capabilities declare its determinism in, " + "whether its output is a function of its inputs and seed; node_key " + "binds the declared value with the rest of the kernel's capabilities" + ), + ("microcosm.graph.kernel", "Numeric"): ( + "the enum a kernel's capabilities declare its numbers' " + "reproducibility in, from which the executor resolves the numeric " + "scope of each column a reader is given; node_key binds the declared " + "value with the rest of the kernel's capabilities" + ), + ("microcosm.graph.kernel", "SeedSource"): ( + "the enum a kernel's capabilities declare its seed source in, where " + "a seeded kernel takes its randomness from; node_key binds the " + "declared value with the rest of the kernel's capabilities" + ), + } +) @dataclass(frozen=True) @@ -1158,6 +1251,77 @@ def engine_reach( return reached methods = more + def keyed_walk( + self, + kernel, + *, + halts: Mapping[tuple[str, str], str] | None = None, + handed_classes: Iterable[tuple[str, str]] = (), + handed_values: Iterable[tuple[str, str]] = (), + key: Iterable[str] = (), + ) -> KeyedWalk: + """What the kernel rule holds one kernel's key to: the walk from what + it runs (``kernel_walk``), with what the objects its build hands it + compute through, and the modules its implementation hash hashes + (``hashed_modules``) with ``key``, those its key binds beyond them. + + A handed value is a definition the build hands the kernel, such as a + gate registry, which the walk follows from its definition as from + what the kernel names. A handed class is the class of an object the + build hands the kernel whose class its code need not name, such as a + rules engine: from the methods the walks so far name on any receiver, + the walk follows them, the constructor the build calls and every + method they call on ``self``, and the methods its own reach names + outside the object's classes, to a fixed point, as the engine root + does (``engine_reach``). Each is walked on its own with the kernel's + ``halts`` (``stops`` unless given) and joins the kernel's walk, its + stops, listed stops met and core names included, so it can only add + to what the kernel walk finds.""" + + halts = self.tables.stops if halts is None else halts + own = self.kernel_walk(type(kernel), halts=halts) + parts = [(Walk(own.reach, own.stops, own.core), own.met, own.core_names)] + for value in handed_values: + module, name = value + node = module_definitions(module, self.scope).definitions[name] + parts.append( + self.walk( + {value}, + [(found, value) for found in self.named(module, node)], + halts=halts, + ) + ) + named = functools.reduce(merged, (walk for walk, _, _ in parts)) + for ref in handed_classes: + handed = class_named(ref) + methods = self.engine_methods(handed, named) + while True: + part = self.walk( + set(self.engine_roots(handed)), + self.engine_seeds(handed, methods), + halts=halts, + ) + more = methods | self.engine_methods_beyond(handed, part[0]) + if more == methods: + break + methods = more + parts.append(part) + walk = functools.reduce(merged, (walk for walk, _, _ in parts)) + met: dict[tuple[str, str], frozenset] = {} + for _, part_met, _ in parts: + for item, namers in part_met.items(): + met[item] = met.get(item, frozenset()) | namers + return KeyedWalk( + KernelWalk( + walk.reach, + walk.stops, + walk.core, + met, + frozenset().union(*(names for _, _, names in parts)), + ), + hashed_modules(kernel) | frozenset(key), + ) + def refusal_only_violations(self) -> list[tuple[str, str]]: """The listed refusal-only checks (``refusal_only``) whose definition can return a value or write (``writes_nothing``).""" diff --git a/test_support/microcosm_build/first_party_imports.py b/test_support/microcosm_build/first_party_imports.py index 105b79b360..e83488c5d0 100644 --- a/test_support/microcosm_build/first_party_imports.py +++ b/test_support/microcosm_build/first_party_imports.py @@ -26,7 +26,7 @@ import importlib import importlib.util import inspect -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from collections.abc import Set as AbstractSet from pathlib import Path from types import ModuleType @@ -206,109 +206,17 @@ def unlisted_imports(listed: Sequence[str], excluded: AbstractSet[str]) -> set[s } -def unhashed_imports( - hashed: AbstractSet[str], *, listed: Sequence[str], excluded: AbstractSet[str] -) -> set[str]: - """``importer -> imported`` for each import a kernel's hash leaves uncovered. - - The walk starts at every module the kernel hashes. An import is covered - when the kernel hashes it, or when the importer is in the shared list - ``listed`` and the import is one of that list's reviewed exclusions - ``excluded``. The walk goes on through every covered import, so an - excluded module's own imports must be covered too, and its imports are - not covered by the list's exclusions, whose reasons were written for the - listed importers alone. An uncovered import is reported and not followed. - """ - - listed_names = set(listed) - uncovered: set[str] = set() - pending = sorted(hashed) - seen: set[str] = set() - while pending: - name = pending.pop() - if name in seen: - continue - seen.add(name) - for imported in sorted(first_party_imports(name)): - if imported in hashed or (name in listed_names and imported in excluded): - pending.append(imported) - else: - uncovered.add(f"{name} -> {imported}") - return uncovered - - class HashInput(NamedTuple): """A module a kernel's file imports only to compute the hash itself. - The exemption holds for the named importers alone, so a hashed module - that starts calling it at run time is still reported. + The exemption holds for the named importers alone: the compile-list + tests check that each still imports it, and which kernels may hash it. """ importers: tuple[str, ...] reason: str -def recorded_import_mismatches( - kernel_groups: Mapping[str, Sequence[tuple[object, ModuleType]]], - *, - listed: Sequence[str], - excluded: AbstractSet[str], - hash_inputs: Mapping[str, HashInput], - recorded: Mapping[str, Sequence[str]], -) -> dict[str, object]: - """Where the walk of each kernel group disagrees with the record. - - ``kernel_groups`` maps a group name to its ``(kernel, module)`` pairs; - every kernel in a group must reach the same unhashed imports, because - the group shares one hash. ``recorded`` maps each ``importer -> - imported`` edge to the groups that leave it unhashed, so removing a - module from one group's hash fails even when another group records the - same edge. Every ``hash_inputs`` edge must be met by some group, and no - kernel may hash a hash input. An empty result means the record holds. - """ - - allowed = { - f"{importer} -> {imported}" - for imported, hash_input in hash_inputs.items() - for importer in hash_input.importers - } - mismatches: dict[str, object] = {} - unknown = { - edge: sorted(set(groups) - set(kernel_groups)) - for edge, groups in recorded.items() - if set(groups) - set(kernel_groups) - } - if unknown: - mismatches["recorded for an unknown group"] = unknown - allowed_met: set[str] = set() - for group, kernels in kernel_groups.items(): - walks = [] - for kernel, module in kernels: - hashed = hashed_module_names(kernel, module) - if set(hash_inputs) & hashed: - mismatches[f"{kernel.ref} hashes a hash input"] = sorted( - set(hash_inputs) & hashed - ) - walks.append(unhashed_imports(hashed, listed=listed, excluded=excluded)) - if any(walk != walks[0] for walk in walks): - mismatches[f"{group}: its kernels reach different imports"] = sorted( - set().union(*walks) - set.intersection(*walks) - ) - met = set().union(*walks) - allowed_met |= met & allowed - met -= allowed - expected = {edge for edge, groups in recorded.items() if group in groups} - if met - expected: - mismatches[f"{group}: unhashed and not recorded"] = sorted(met - expected) - if expected - met: - mismatches[f"{group}: recorded but no longer met (delete them)"] = sorted( - expected - met - ) - if allowed - allowed_met: - mismatches["hash inputs no importer reaches"] = sorted(allowed - allowed_met) - return mismatches - - def hashed_module_names(kernel: object, module: ModuleType) -> frozenset[str]: """Every module ``kernel.implementation_hash()`` passes to ``source_hash``. diff --git a/test_support/microcosm_build/transport_composed.py b/test_support/microcosm_build/transport_composed.py index 9b1d3b8b44..7991b2f82a 100644 --- a/test_support/microcosm_build/transport_composed.py +++ b/test_support/microcosm_build/transport_composed.py @@ -915,6 +915,21 @@ def write_rulespec_tree(root: Path) -> list[dict]: return rows +def toy_rules_bindings(rows: list[dict]) -> dict: + """The toy package's ``axiom_rules_bindings`` document over the binding + ``rows`` ``write_rulespec_tree`` returns, with format-valid synthetic + pins.""" + return { + "engine": {"commit": "a" * 40, "wheel_sha256": "b" * 64}, + "rulespec": {"commit": "c" * 40}, + "entity_names": {entity: entity.capitalize() for entity in ENTITIES}, + "periods": { + PERIOD: {"kind": "tax_year", "start": "2026-04-01", "end": "2027-03-31"} + }, + "bindings": rows, + } + + @dataclass(frozen=True) class ComposedFixture: """A written toy package, its inputs and one prepared registry.""" @@ -989,15 +1004,7 @@ def make_composed_fixture(root: Path) -> ComposedFixture: mapping = _concept_mapping() rules_root = root / "rulespec" rows = write_rulespec_tree(rules_root) - bindings = { - "engine": {"commit": "a" * 40, "wheel_sha256": "b" * 64}, - "rulespec": {"commit": "c" * 40}, - "entity_names": {entity: entity.capitalize() for entity in ENTITIES}, - "periods": { - PERIOD: {"kind": "tax_year", "start": "2026-04-01", "end": "2027-03-31"} - }, - "bindings": rows, - } + bindings = toy_rules_bindings(rows) closure = InputClosure( country=COUNTRY, content_basis=ContentBasis.TRANSPORT, diff --git a/test_support/microcosm_build/transport_kernels.py b/test_support/microcosm_build/transport_kernels.py new file mode 100644 index 0000000000..7e423722e7 --- /dev/null +++ b/test_support/microcosm_build/transport_kernels.py @@ -0,0 +1,64 @@ +"""Every kernel the transport registry registers, built engine-free. + +``build_transport_registry`` prepares one rules engine per binding before it +registers its kernels, which needs a RuleSpec tree and an engine build. The +kernels' implementation hashes and the derived binding rule's walks read only +the engines' class, so ``transport_kernels`` replays the registry's +registration calls with a stand-in for the prepared engines: an +``AxiomEngine``, the class the registry builds, created without a program. +test_transport_binding_rule holds the replay to the registry itself. +""" + +from __future__ import annotations + +import functools +from collections.abc import Mapping +from types import MappingProxyType + +from microcosm.build.graph_atomic_geography import register_atomic_geography_kernels +from microcosm.build.transport.codecs import register_transport_codecs +from microcosm.build.transport.evidence_kernels import register_evidence_kernels +from microcosm.build.transport.gate_bindings import TRANSPORT_GATE_REGISTRY +from microcosm.build.transport.gate_kernels import register_gate_kernel +from microcosm.build.transport.person_location_kernels import ( + register_person_location_kernels, +) +from microcosm.build.transport.registry import register_transport_population_kernels +from microcosm.build.transport.reporting_kernels import register_reporting_kernels +from microcosm.build.transport.takeup_kernels import register_takeup_kernels +from microcosm.build.transport.target_kernels import register_target_kernels +from microcosm.build.transport.terminal_kernels import register_terminal_kernels +from microcosm.calibrate.ordered_kernels import CALIBRATE_ORDERED_ADAM +from microcosm.frame.adapters.axiom import AxiomEngine +from microcosm.frame.rules_kernels import ( + SimulateCounterfactualKernel, + SimulateRulesByRefKernel, + SimulateSolveZeroKernel, +) +from microcosm.graph import KernelRegistry +from microcosm.graph.codecs import SourceCodecRegistry + + +@functools.cache +def transport_kernels() -> Mapping[str, object]: + """Every kernel ``build_transport_registry`` registers, by ref, from its + registration calls with one stand-in engine.""" + + engines = MappingProxyType({"stand-in": object.__new__(AxiomEngine)}) + kernels = KernelRegistry() + codecs = SourceCodecRegistry() + register_transport_codecs(codecs) + register_transport_population_kernels(kernels) + register_atomic_geography_kernels(kernels) + register_person_location_kernels(kernels) + register_target_kernels(kernels) + kernels.register(CALIBRATE_ORDERED_ADAM) + register_gate_kernel(kernels, TRANSPORT_GATE_REGISTRY) + register_terminal_kernels(kernels, codecs=codecs) + kernels.register(SimulateRulesByRefKernel(engines)) + kernels.register(SimulateCounterfactualKernel(engines)) + kernels.register(SimulateSolveZeroKernel(engines)) + register_takeup_kernels(kernels) + register_reporting_kernels(kernels) + register_evidence_kernels(kernels, engines) + return MappingProxyType({ref: kernels.get(ref) for ref in kernels.refs()}) From 358cc044110991cbbeeff30f996d16074e25a07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:19:33 +0000 Subject: [PATCH 047/108] Close two gaps in the refusal check's local containers and control the shared checks (#931) The review of 354296a59 and 8c10ecc22 found that the extraction moves nothing for the UK, and four gaps in the refusal check and its controls. This commit closes them. It changes the shared machinery and its controls only: writes_nothing checks only the listed refusal-only entries, so no table, walk, verdict or key moves. - Augmenting a local container as a whole now needs a new container as the value. x += v can rebind x to whatever v's reflected operator returns, v itself included: Python tries that operator before a list's in-place concatenation, and after a set's in-place operator declines a value that is not a set. So seen = []; seen += values; seen[0] = 1 wrote into values and passed. 8c10ecc22's message and docstring said a container's augmented operators change it in place or refuse; they need not. - An in-place call on a local container is a write again when it is handed another object's in-place method: seen.sort(key=values.pop) and seen.extend(iter(values.pop, None)) empty values. Both were refused before 8c10ecc22. A method handed to any other call, as in sorted(rows, key=values.pop), was not seen before 8c10ecc22 and still is not; the docstring now says so. - A global or nonlocal declaration disqualifies a name, as 8c10ecc22's message said every binding does. writes_nothing already refused such a function, so no verdict moves. - The controls: - the containers control gains a writing case for each branch that had none: an import, a nested function, an exception capture, a match capture, a class body (which the function's own scope leaves out), a global declaration, the two augmentations by a handed value and the two handed methods; and one more refusing case, a set joined by a new set; - test_the_shared_checks_report_what_they_find plants a violation for each check 354296a59 moved into the shared module (refusal_only_violations, core_modules_off_run_graph_path and findings), and holds the per-rule cache of each definition's kind: two rules whose tables differ classify the same alias each by its own tables, whichever classifies first. Corrections to the earlier messages, which stay as they are: - 354296a59 says four UK test bodies now call the shared checks; five do. It also says each moved control builds a rule from its own scratch tables instead of monkeypatching the UK's: the refusal-check control builds no rule, and only the helper-rule and kernel-rule controls monkeypatched the UK's tables before. - 8c10ecc22 says bindings of every kind count against a local container. Global and nonlocal declarations did not until this commit, and its list of the other bindings leaves out the augmented assignments and bare annotations it accepts. Tests: all code facts. - test_derived_binding_rule passes (7 tests). - The six UK and two transport refusal-only entries still pass, and uk_runtime.ledger_targets._assert_household_type_bindings_declared still passes the check. - Eleven mutations of the shared module each fail a new control: removing the import, nested-definition, capture or global branch, letting class bodies into a function's own scope, dropping the augmented values or the callback check, emptying refusal_only_violations, core_modules_off_run_graph_path or findings' stops, and sharing the kind cache across rules. - Ruff is clean. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_derived_binding_rule.py | 151 ++++++++++++++++-- .../microcosm_build/derived_binding_rule.py | 60 +++++-- 2 files changed, 185 insertions(+), 26 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py b/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py index 614354dd10..ee1ff1fba2 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_derived_binding_rule.py @@ -20,10 +20,15 @@ import microcosm from test_support.microcosm_build.derived_binding_rule import ( FIRST_PARTY_SCOPE, + FRAME_SCOPE, LAZY_EXPORT_TABLES, NO_LAZY_EXPORTS, BindingRule, + Findings, + KernelWalk, + KeyedWalk, RuleTables, + findings, hashed_modules, local_containers, module_definitions, @@ -63,14 +68,18 @@ def test_the_refusal_check_sees_each_way_a_function_writes() -> None: def test_the_refusal_check_lets_a_function_write_into_containers_it_creates(): """Control for ``local_containers`` (microcosm#931): a function that - stores into, deletes from, augments or calls an in-place method on a - container its own scope creates, and that no other binding reaches, - writes nothing outside itself, a closure over it included. A store into - an item of an item or an attribute of one, an augmented store into an - item (``seen = list(rows)`` holds the rows' own objects), a store through - a name bound otherwise anywhere in the function (an alias, a parameter, a - loop target or an unpacked binding), one through a name a nested scope - binds, which in the function is a global, a container built by a + stores into, deletes from, augments with a new container or calls an + in-place method on a container its own scope creates, and that no other + binding reaches, writes nothing outside itself, a closure over it + included. A store into an item of an item or an attribute of one, an + augmented store into an item (``seen = list(rows)`` holds the rows' own + objects), a store through a name bound otherwise anywhere in the function + (an alias, a parameter, a loop target, an unpacked binding, an import, a + nested definition, an exception or match capture, or a global + declaration), one through a name only a nested scope binds (a class body + included), which in the function is a global, an augmentation by a + handed value, whose reflected operator can return itself, an in-place + call handed a parameter's in-place method, a container built by a ``dict`` the function binds, and returning the container still count.""" refusing = { "local dict": ( @@ -114,6 +123,9 @@ def test_the_refusal_check_lets_a_function_write_into_containers_it_creates(): " if bad:\n" " raise ValueError(bad)\n" ), + "local set joined by a new set": ( + "def f(rows):\n seen = set()\n seen |= {row for row in rows}\n" + ), "local item deleted": ( "def f(rows):\n" " seen = {row: 1 for row in rows}\n" @@ -174,6 +186,49 @@ def test_the_refusal_check_lets_a_function_write_into_containers_it_creates(): "bound only in a nested scope": ( "def f(rows):\n def note():\n seen = {}\n\n seen['a'] = 1\n" ), + "bound only in a class body": ( + "def f(rows):\n class Holder:\n seen = {}\n\n seen['a'] = 1\n" + ), + "rebound by an import": ( + "def f(rows):\n seen = {}\n from m import TABLE as seen\n seen['a'] = 1\n" + ), + "rebound by a nested function": ( + "def f(rows):\n" + " seen = {}\n\n" + " def seen():\n" + " pass\n\n" + " seen['a'] = 1\n" + ), + "rebound by an exception capture": ( + "def f(rows):\n" + " seen = {}\n" + " try:\n" + " rows[0]\n" + " except LookupError as seen:\n" + " seen['a'] = 1\n" + ), + "rebound by a match capture": ( + "def f(values):\n" + " seen = {}\n" + " match values:\n" + " case seen:\n" + " seen['a'] = 1\n" + ), + "declared global": ( + "def f(rows):\n global seen\n seen = {}\n seen['a'] = 1\n" + ), + "augmented by a handed value": ( + "def f(values):\n seen = []\n seen += values\n seen[0] = 1\n" + ), + "joined by a handed value": ( + "def f(values):\n seen = set()\n seen |= values\n seen.add(1)\n" + ), + "handed a parameter's method": ( + "def f(rows, values):\n seen = list(rows)\n seen.sort(key=values.pop)\n" + ), + "handed a parameter's method within a call": ( + "def f(values):\n seen = []\n seen.extend(iter(values.pop, None))\n" + ), } for label, source in {**refusing, **writing}.items(): function = ast.parse(source).body[0] @@ -181,8 +236,84 @@ def test_the_refusal_check_lets_a_function_write_into_containers_it_creates(): # The containers it counts as the function's own. function = ast.parse(refusing["closure over a local"]).body[0] assert local_containers(function) == {"seen"} - function = ast.parse(writing["unpacked binding"]).body[0] - assert local_containers(function) == frozenset() + for label in ("unpacked binding", "declared global", "augmented by a handed value"): + function = ast.parse(writing[label]).body[0] + assert local_containers(function) == frozenset(), label + + +def test_the_shared_checks_report_what_they_find(tmp_path, monkeypatch) -> None: + """Control for the checks the countries share (microcosm#931): each + reports what is planted for it. ``refusal_only_violations`` names each + listed check, a kernel's method included, that writes; + ``core_modules_off_run_graph_path`` a listed core module the walk from + run_graph never meets; ``findings`` each walk's modules outside its key, + the stops outside the key, the listed stops met and the core names read + outside the Frame's modules. Two rules whose tables differ classify the + same alias each by its own tables, whichever classifies first.""" + scratch = f"shared_checks_{uuid.uuid4().hex[:8]}" + package = tmp_path / "microcosm" / scratch + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + (package / "checks.py").write_text( + "def refuses(frame):\n" + " if frame is None:\n" + " raise ValueError('no frame')\n\n" + "def writes(frame):\n" + " frame[0] = 1\n\n" + "class Kernel:\n" + " def check(self, frame):\n" + " frame.clear()\n" + ) + (package / "decls.py").write_text("TABLE = {'a': 1}\n") + (package / "aliases.py").write_text("from .decls import TABLE\n\nALIAS = TABLE\n") + monkeypatch.syspath_prepend(str(tmp_path)) + prefix = f"microcosm.{scratch}." + rule = BindingRule( + RuleTables( + core_modules={ + "microcosm.graph.executor": "control", + f"{prefix}decls": "control", + }, + refusal_only={ + (f"{prefix}checks", "refuses"): "control", + (f"{prefix}checks", "writes"): "control", + (f"{prefix}checks", "Kernel.check"): "control", + }, + ) + ) + assert rule.refusal_only_violations() == [ + (f"{prefix}checks", "Kernel.check"), + (f"{prefix}checks", "writes"), + ] + assert rule.core_modules_off_run_graph_path() == [f"{prefix}decls"] + + walk = KernelWalk( + reach={"m.keyed": frozenset({"a"}), "m.missed": frozenset({"b"})}, + stops=frozenset({("m.keyed", "NAME"), ("m.other", "NAME")}), + core=frozenset(), + met={("m.stop", "STOP"): frozenset({"a"})}, + core_names=frozenset( + {("microcosm.graph.decl", "ArtifactType"), (f"{FRAME_SCOPE}x", "Frame")} + ), + ) + assert findings({"k@1": {"k@1": KeyedWalk(walk, frozenset({"m.keyed"}))}}) == ( + Findings( + {"k@1": ["m.missed"]}, + frozenset({("m.other", "NAME")}), + frozenset({("m.stop", "STOP")}), + frozenset({("microcosm.graph.decl", "ArtifactType")}), + ) + ) + + alias = (f"{prefix}aliases", "ALIAS") + declared = RuleTables(core_modules={}, declaration_modules=(f"{prefix}decls",)) + undeclared = RuleTables(core_modules={}) + for order in ((declared, undeclared), (undeclared, declared)): + kinds = { + tables.declaration_modules: BindingRule(tables).helper_kind(*alias) + for tables in order + } + assert kinds == {(f"{prefix}decls",): "name", (): "value"} def test_the_helper_rule_follows_each_way_a_stage_names_a_helper( diff --git a/test_support/microcosm_build/derived_binding_rule.py b/test_support/microcosm_build/derived_binding_rule.py index f5e2c3f1d1..3474275276 100644 --- a/test_support/microcosm_build/derived_binding_rule.py +++ b/test_support/microcosm_build/derived_binding_rule.py @@ -397,20 +397,25 @@ def local_containers(node) -> frozenset[str]: dict, list or set display or comprehension, or a call of ``dict``, ``list`` or ``set``), every other binding of it anywhere in the function, nested scopes included, is one too, and it is a parameter of neither the - function nor anything nested in it. A bare annotation binds nothing, and - an augmented assignment to the name itself keeps it local, since a - container's augmented operators change it in place or refuse. A call - counts only where the function binds no name ``dict``, ``list`` or - ``set`` of its own; one its module binds at its top level is not seen.""" + function nor anything nested in it, nor declared ``global`` or + ``nonlocal`` in either. A bare annotation binds nothing. An augmented + assignment to the name itself is a binding whose value must be a new + container too: ``x += v`` can rebind ``x`` to whatever ``v``'s reflected + operator returns, ``v`` itself included, since Python tries that operator + before a list's in-place concatenation, and after a set's in-place + operator declines a value that is not a set. A call counts only where the + function binds no name ``dict``, ``list`` or ``set`` of its own; one its + module binds at its top level is not seen.""" parameters: set[str] = set() for sub in ast.walk(node): if isinstance(sub, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda): parameters |= _parameters(sub) - # Each single-name binding with its value, and the name targets they and - # the bindings of no new object (a bare annotation, an augmented - # assignment to a name) account for. + # Each single-name binding with its value, each augmented assignment to a + # name with its value, and the name targets they and a bare annotation, + # which binds nothing, account for. valued: dict[int, tuple[str, ast.expr]] = {} + augmented: list[tuple[str, ast.expr]] = [] accounted: set[int] = set() for sub in ast.walk(node): if ( @@ -425,10 +430,11 @@ def local_containers(node) -> frozenset[str]: valued[id(sub)] = (sub.target.id, sub.value) accounted.add(id(sub.target)) elif isinstance(sub, ast.AugAssign) and isinstance(sub.target, ast.Name): + augmented.append((sub.target.id, sub.value)) accounted.add(id(sub.target)) # Every other binding: loop, with, walrus, unpacking and comprehension # targets, deletions, imports, nested definitions, exception and match - # captures. + # captures, and global and nonlocal declarations. other: set[str] = set() for sub in ast.walk(node): if isinstance(sub, ast.Name) and isinstance(sub.ctx, ast.Store | ast.Del): @@ -446,6 +452,8 @@ def local_containers(node) -> frozenset[str]: other.add(sub.name) elif isinstance(sub, ast.MatchMapping) and sub.rest is not None: other.add(sub.rest) + elif isinstance(sub, ast.Global | ast.Nonlocal): + other.update(sub.names) stored = { sub.id for sub in ast.walk(node) @@ -459,7 +467,9 @@ def local_containers(node) -> frozenset[str]: if key in own and _fresh_container(value, calls) } stale = { - name for name, value in valued.values() if not _fresh_container(value, calls) + name + for name, value in (*valued.values(), *augmented) + if not _fresh_container(value, calls) } return frozenset(fresh - stale - other - parameters) @@ -488,6 +498,19 @@ def _item_of(target, local: frozenset[str]) -> bool: ) +def _hands_in_place_method(call, local: frozenset[str]) -> bool: + """An argument of ``call`` names an in-place method of anything but a + local container, which the call can run: ``seen.sort(key=values.pop)``.""" + + return any( + isinstance(inner, ast.Attribute) + and inner.attr in IN_PLACE_CALLS + and not (isinstance(inner.value, ast.Name) and inner.value.id in local) + for argument in (*call.args, *(keyword.value for keyword in call.keywords)) + for inner in ast.walk(argument) + ) + + def writes_nothing(node) -> bool: """Whether a function can only refuse: it returns nothing but None, or its sole parameter unrebound, and writes nothing outside itself: no store @@ -497,11 +520,14 @@ def writes_nothing(node) -> bool: A container the function creates is its own (``local_containers``): it may store into or delete one item of it, call an in-place method on it, - or augment it as a whole. A nested store (into an item of an item, or an - attribute of one), an augmented store into an item, which can change in - place an object the function was handed, and a store through any other - name, an alias of the container included, are still writes, and - returning the container returns a value.""" + or augment it as a whole by a new container. A nested store (into an + item of an item, or an attribute of one), an augmented store into an + item, which can change in place an object the function was handed, a + store through any other name, an alias of the container included, and + an in-place call on it that is handed another object's in-place method + (``seen.sort(key=values.pop)``) are still writes, and returning the + container returns a value. A method handed to any other call + (``sorted(rows, key=values.pop)``) is not seen.""" parameters = _parameters(node) rebound = { @@ -546,7 +572,9 @@ def writes_nothing(node) -> bool: isinstance(function, ast.Attribute) and function.attr in IN_PLACE_CALLS and not ( - isinstance(function.value, ast.Name) and function.value.id in local + isinstance(function.value, ast.Name) + and function.value.id in local + and not _hands_in_place_method(sub, local) ) ): return False From 0b888f4b0c825f028157c392dea882c567085af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:25:19 +0000 Subject: [PATCH 048/108] Hold the engine hand-off to the built registry and correct two transport reasons (#931) The review of 6c76c90a7 found no binding the code misses today. It found one way a new kernel could get past the transport's tables, two reasons that were wrong, and claims in the changelog fragment and the message that overstate. This commit addresses them. It changes the transport rule test and the changelog fragment only: two reason texts change, and no table entry is added or dropped, no walk moves and no key moves. - The kernels the registry hands its engines to are now held to the registry as built, not only to its code's spelling. test_the_registry_hands_the_kernels_what_the_tables_list matched only a positional argument named engine_mapping, so a new kernel handed the engines by keyword, whose run calls their methods, passed every test while its key hashed only its own module. The test now: - builds build_transport_registry with the toy package's engines (a module-scoped fixture the registration test shares), and requires the kernels that hold one of those engines, at any depth of their attributes, mappings, collections, bound methods, partials or closures, to be exactly the four HANDED_CLASSES lists; - scans keyword arguments as well as positional ones for the mapping; - requires AxiomEngine to be the only class with the rules-engine protocol's methods that the builder constructs, so a second engine class built for some bindings cannot go unwalked. The review's probe, a kernel handed the engines by keyword and registered in both the registry and the replay, now fails the test, as does the same kernel handed them through **{"engines": engine_mapping}, which only the built-registry check sees. - Two reasons are corrected: - the Frame's said no transport key hashes it. The matrix kernels, the ordered solve, the battery, the rules kernels and export hash its modules for their own reasons; it now says the rule asks no kernel to hash them; - require_outputs' said only the kernels that write the problem's and the diagnostics' outputs compute through those outputs' modules. The kernels that read them do too (the battery, the ordered solve and the weights audit), and the rule holds every one of them to hash those modules. - The changelog fragment's first sentence now says the kernels hash what they compute through as far as the rule can see, apart from the core modules and the two refusal-only checks, and its New Zealand count says the test composition's 18 synthetic quantile stand-ins re-key too. Two bindings are wider than the kernels need, and stay, since they cost keys only: simulate.counterfactual@1 now hashes frame.exact_money, which only the zero solve's quantized branches run, so an edit to it re-keys the counterfactual's nodes needlessly; and the input closure hashes frame.materialize because the rule follows AxiomEngine whole. Corrections to the earlier messages, which stay as they are: - 6c76c90a7 says all 357 nodes of New Zealand's packaged graph re-key. Composed as test_nz_transport_declarations composes it, the graph has 375 nodes, 18 of them synthetic quantile stand-ins, and all 375 re-key. - 6c76c90a7 says each table entry has a reason a test checks; the two reasons above were neither true nor checked. - 8f6c36917 says 8c10ecc22's message and docstring claimed a container's augmented operators change it in place or refuse; only its docstring did. Tests: all code facts. - test_transport_binding_rule passes (8 tests). Ruff is clean. - The review's probe, with the engines handed by keyword and positionally, and its **-mapping variant each fail the test. Co-Authored-By: Claude Opus 5.5 --- .../931-transport-helper-bindings.changed.md | 4 +- .../shared/test_transport_binding_rule.py | 126 ++++++++++++++---- 2 files changed, 103 insertions(+), 27 deletions(-) diff --git a/changelog.d/931-transport-helper-bindings.changed.md b/changelog.d/931-transport-helper-bindings.changed.md index c62451e354..b010ec8aed 100644 --- a/changelog.d/931-transport-helper-bindings.changed.md +++ b/changelog.d/931-transport-helper-bindings.changed.md @@ -1,4 +1,4 @@ -Every transport kernel now hashes each first-party module its own code, or an object the transport registry hands it, computes through, under the derived binding rule the UK stages and kernels are held to (#931). `source_hash` hashes only the files it is given, so a transport node that called a function in a module its hash did not name replayed a stale cached output when that module changed. Only node keys move. +Every transport kernel now hashes each first-party module that its own code, or an object the transport registry hands it, computes through, as far as the derived binding rule the UK stages and kernels are held to can see, apart from the core modules every node runs through and two checks that only refuse (#931). `source_hash` hashes only the files it is given, so a transport node that called a function in a module its hash did not name replayed a stale cached output when that module changed. Only node keys move. - **The rule, shared.** The rule's walk moved from `test_uk_graph` to `test_support/microcosm_build/derived_binding_rule.py`. Each country passes its tables as a declaration there, and the UK's walks are unchanged. The rule's refusal check now counts a store into a container the function itself creates as local. - **The transport's tables.** `test_transport_binding_rule` holds the 45 kernels `build_transport_registry` registers to the rule: the transport package's 37, the shared atomic geography's four, the ordered solve and the three rules kernels. The transport has its own tables, each entry with a reason a test checks: the eight core modules, 19 core definitions, two refusal-only checks (`graph_inputs.require_outputs` and `reference_profiles._validate_interval_resolution`) and one stop, `target_kernels._COMPILE_MODULES`, which only implementation hashes read. The rule also walks what the registry hands the kernels: the `AxiomEngine` behind the rules kernels and the input closure, and the gate registry behind the battery, whose key also binds its bindings' source closure. @@ -8,4 +8,4 @@ Every transport kernel now hashes each first-party module its own code, or an ob - `artifact_types`, `calibrate.target`, `graph_atomic_geography` and the Axiom input surface where kernels name them; the gate report records (`country_spec`, `gate_battery`, `gates`) in the export and package kernels; `build.trace` in the export readback; `takeup_kernels` in the battery; `frame.rules` and `frame.materialize` in the input closure; `calibrate.hierarchy` and `monetary_binding` where targets or matrices are rebuilt; `frame.exact_money` in the counterfactual bridge, as in the zero solve; - the ordered solve's internals (exact-k, the L0 gates and their initialization record, the hierarchy, the monetary binding and the encoder) in `calibrate.ordered_adam@1`'s own half, so `calibrate.adam@1`'s hash, which UK kernels and the parity pins embed, does not move. - **The import record.** `test_transport_compile_modules` drops microcosm#1497's record of unclassified imports, its kernel groups and the record check. It keeps the compile list's closure checks and the geography-label re-key test, and now checks that no transport kernel hashes the compile list. The import scan's record helpers, which nothing else used, go. -- **What moves.** Node keys only. 40 of the 45 transport kernel hashes move; the four atomic geography kernels and the person location keep theirs. All 357 nodes of New Zealand's packaged graph re-key: 353 directly, and the atomic geography and person location nodes by chaining. Belgium composes no graph. No UK kernel hash moves (0 of 99), nor do `calibrate.adam@1`'s or `simulate.rules@1`'s, and nothing of the US's. No value, declaration or generated file moves. +- **What moves.** Node keys only. 40 of the 45 transport kernel hashes move; the four atomic geography kernels and the person location keep theirs. All 357 nodes of New Zealand's packaged graph re-key: 353 directly, and the atomic geography and person location nodes by chaining; the 18 synthetic quantile stand-ins its test composition adds re-key too. Belgium composes no graph. No UK kernel hash moves (0 of 99), nor do `calibrate.adam@1`'s or `simulate.rules@1`'s, and nothing of the US's. No value, declaration or generated file moves. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_binding_rule.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_binding_rule.py index 0462472ddb..b6fca5a3b5 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_binding_rule.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_binding_rule.py @@ -36,10 +36,14 @@ from __future__ import annotations import ast +import contextlib import functools -from collections.abc import Callable +import importlib +from collections.abc import Callable, Mapping from pathlib import Path -from types import MappingProxyType +from types import FunctionType, MappingProxyType, MethodType, ModuleType + +import pytest import microcosm.build.transport as transport_package from microcosm.build import graph_atomic_geography @@ -56,6 +60,7 @@ from microcosm.build.transport.registry import build_transport_registry from microcosm.calibrate.ordered_kernels import CalibrateOrderedAdamKernel from microcosm.frame import rules_kernels +from microcosm.frame.rules import RulesEngine from microcosm.frame.rules_kernels import ( SimulateCounterfactualKernel, SimulateRulesByRefKernel, @@ -83,9 +88,12 @@ #: What catches an edit to the Frame, whose three modules share it. _FRAME_REASON = ( "the kernels compute on frames, and the executor holds every population " - "version in one. No transport key hashes it; the transport graph tests, " - "which run composed graphs through run_graph (test_transport_composed_graph " - "and test_nz_transport_toy_graph among them), catch an edit that changes a " + "version in one. The rule asks no transport kernel to hash it, and an " + "edit to it re-keys only the kernels whose keys name it for their own " + "reasons (the matrix kernels, the ordered solve, the battery, the rules " + "kernels and export among them); the transport graph tests, which run " + "composed graphs through run_graph (test_transport_composed_graph and " + "test_nz_transport_toy_graph among them), catch an edit that changes a " "value on their fixtures, and the Frame's own tests the rest" ) #: The core modules every node runs through, which the rule neither follows @@ -197,8 +205,10 @@ "(artifact_types.KERNEL_OUTPUTS), and returns nothing. The table it " "reads types the outputs of every kernel, the problem's and the " "diagnostics' among them, whose modules (calibrate.artifacts, " - "diagnostics.schema) only the kernels that write those outputs " - "compute through" + "diagnostics.schema) only the kernels that write or read those " + "outputs compute through (the battery, the ordered solve and the " + "weights audit among the readers), and the rule holds each of those " + "to hash them" ), ("microcosm.build.reference_profiles", "_validate_interval_resolution"): ( "binds recorded interval scaling to the selected factors, and to the " @@ -332,7 +342,58 @@ def _package_sources(package) -> list[tuple[str, Path]]: ] -def test_the_rule_sees_every_kernel_the_transport_registers(tmp_path) -> None: +@pytest.fixture(scope="module") +def toy_registry(tmp_path_factory): + """build_transport_registry itself, built engine-free with the toy + package's engines, and the engines it was handed, by binding id.""" + + root = tmp_path_factory.mktemp("rulespec") + rows = write_rulespec_tree(root) + engines = {row["id"]: ToyRulesEngine(root, row) for row in rows} + prepared = build_transport_registry( + toy_rules_bindings(rows), root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) + return prepared, MappingProxyType(engines) + + +def _holds(root, targets: frozenset[int]) -> bool: + """Whether ``root`` holds an object whose id is in ``targets``, at any + depth: through its attributes, slots included, mappings, collections, + bound methods, partials and closures.""" + + pending, seen = [root], set() + while pending: + item = pending.pop() + if id(item) in targets: + return True + if id(item) in seen or isinstance( + item, type | ModuleType | str | bytes | int | float + ): + continue + seen.add(id(item)) + if isinstance(item, Mapping): + pending.extend((*item.keys(), *item.values())) + elif isinstance(item, tuple | list | set | frozenset): + pending.extend(item) + elif isinstance(item, functools.partial): + pending.extend((item.func, *item.args, *item.keywords.values())) + elif isinstance(item, MethodType): + pending.extend((item.__self__, item.__func__)) + elif isinstance(item, FunctionType): + for cell in item.__closure__ or (): + with contextlib.suppress(ValueError): + pending.append(cell.cell_contents) + else: + pending.extend(getattr(item, "__dict__", {}).values()) + for cls in type(item).__mro__: + slots = getattr(cls, "__slots__", ()) + for name in (slots,) if isinstance(slots, str) else slots: + with contextlib.suppress(AttributeError): + pending.append(getattr(item, name)) + return False + + +def test_the_rule_sees_every_kernel_the_transport_registers(toy_registry) -> None: """The kernels the rule walks are every kernel class the transport package defines, all registered, and the eight the registry registers from elsewhere: the shared atomic geography's four, the ordered solve and the @@ -351,14 +412,7 @@ def test_the_rule_sees_every_kernel_the_transport_registers(tmp_path) -> None: SimulateCounterfactualKernel, SimulateSolveZeroKernel, } - # The registry itself, built engine-free with the toy package's engines. - rows = write_rulespec_tree(tmp_path) - prepared = build_transport_registry( - toy_rules_bindings(rows), - tmp_path, - unit_rule=UNIT_RULE, - engines_by_binding={row["id"]: ToyRulesEngine(tmp_path, row) for row in rows}, - ) + prepared, _ = toy_registry assert { ref: type(prepared.kernels.get(ref)) for ref in prepared.kernels.refs() } == {ref: type(kernel) for ref, kernel in kernels.items()} @@ -463,13 +517,16 @@ def test_the_kernel_rule_stops_only_where_its_reasons_hold() -> None: assert id(sub) in hashing, (module, sub.lineno) -def test_the_registry_hands_the_kernels_what_the_tables_list() -> None: - """build_transport_registry builds AxiomEngine engines and hands their - mapping to exactly the rules kernels and the evidence producers, which - hand it to the input closure alone (it also returns the mapping, in the - TransportRegistry its caller runs the graph with), and builds the battery - with the transport gate registry: the objects ``HANDED_CLASSES`` and - ``HANDED_VALUES`` list for those kernels.""" +def test_the_registry_hands_the_kernels_what_the_tables_list(toy_registry) -> None: + """build_transport_registry builds AxiomEngine engines, and no other + rules-engine class, and hands their mapping to exactly the rules kernels + and the evidence producers, which hand it to the input closure alone (it + also returns the mapping, in the TransportRegistry its caller runs the + graph with), and builds the battery with the transport gate registry: the + objects ``HANDED_CLASSES`` and ``HANDED_VALUES`` list for those kernels. + Built with the toy package's engines, the registry's kernels that hold + one of the engines it was handed, however they hold it, are exactly the + kernels ``HANDED_CLASSES`` lists.""" builder = module_definitions(registry.__name__, FIRST_PARTY_SCOPE).definitions[ "build_transport_registry" ] @@ -487,12 +544,23 @@ def handed(node, module: str, argument: str) -> set: for callee, call in called(node, module) if any( isinstance(value, ast.Name) and value.id == argument - for value in call.args + for value in (*call.args, *(item.value for item in call.keywords)) ) } calls = called(builder, registry.__name__) - assert ENGINE_CLASS in {callee for callee, _ in calls} + protocol = {name for name in vars(RulesEngine) if not name.startswith("_")} + built = { + found + for sub in ast.walk(builder) + if isinstance(sub, ast.Call) + for found in _RULE.named(registry.__name__, sub.func) + if isinstance( + definition := getattr(importlib.import_module(found[0]), found[1]), type + ) + and protocol <= set(dir(definition)) + } + assert built == {ENGINE_CLASS} assert handed(builder, registry.__name__, "engine_mapping") == { (rules_kernels.__name__, "SimulateRulesByRefKernel"), (rules_kernels.__name__, "SimulateCounterfactualKernel"), @@ -513,6 +581,14 @@ def handed(node, module: str, argument: str) -> set: SimulateSolveZeroKernel, evidence_kernels.TransportInputClosureKernel, } + prepared, engines = toy_registry + injected = frozenset(map(id, engines.values())) + holding = { + ref + for ref in prepared.kernels.refs() + if _holds(prepared.kernels.get(ref), injected) + } + assert holding == set(HANDED_CLASSES), sorted(holding ^ set(HANDED_CLASSES)) (battery,) = [ call for callee, call in calls From 099bd160b8a9fee2a8477694dbf42c00ad829e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:25:19 +0000 Subject: [PATCH 049/108] Move the gate phase codes into a refusal-only module and refuse a stale spine phase code (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María ruled on 9 Nov (late) that the gate phase-code functions and tables move into a small module that only the stale-code checks read, listed refusal-only (3a), and that the spine gates refuse a stale or missing phase code, as the full gates do (3b). This commit does both and takes the first two nits the re-review of a0933a56d left. It moves keys only, plus that one refusal. - The module. uk_runtime/gate_phase_codes.py holds the four phase-code functions, with their @cache and docstrings, and the two tables, which graph_evidence and graph_terminal defined. Their code is unchanged, so each phase code keeps its value. It adds one check both batteries call, require_current_phase_code(params, phase, *, battery), which raises ValueError, naming the battery and phase, unless params carries `_code` equal to the phase's code in that battery's table (a phase the table leaves out must carry none). Nothing else is in it. The graph construction imports the tables where it sets the params and the two runs import the check, so neither old module carries the names. - The check takes the battery's name, not its table. A run that named a table would compute through the module by the rule, so its kernel would have to hash it, and an edit to a phase's list would again re-key every node whose kernel hashes the gate kernels' modules. Only the listed check names the tables. - The rule. test_uk_graph lists the check in _REFUSAL_ONLY_HELPERS: it reads its params and the phase's code and refuses, writing nothing, and both gate kernels' runs call it first. The battery walk reads each phase's code table from the new module. source_hash leaves _KERNEL_CORE_NAMES rather than taking a new reason: only the full gates' run named it, through UK_FULL_GATE_PHASE_CODE, and the rule does not follow a listed check, so no kernel's code names it now, and test_the_core_definitions_the_kernels_name_are_listed holds the table to exactly what the walks name. Its reason moves into the check's entry. Two new tests: each gate kernel's run calls the check on its own battery, in a statement of its own, before it reads anything of its node but its parameters, and evaluates the phase it checked; and no UK kernel's key covers gate_phase_codes, the gate kernels' implementation hashes among them, while graph_evidence and graph_terminal carry none of the moved names. - The spine gates. UKSpineGateKernel.run calls the check before it reads or evaluates anything, and its docstring says it refuses a stale or missing code. test_uk_graph_evidence mirrors the full gates' stale-code test: each spine phase refuses a stale and a missing code before any artifact read, and passes its node's own declared code. The check that each spine gate node carries its own phase's code moves there from test_uk_graph's battery test, beside the full gates' in test_uk_graph_terminal. - Nit 1. uk_terminal_only_code_hash's docstring and the changelog said nine terminal-only modules never re-solve, on a bound spine or in the in-graph build. uk_runtime.diagnostics re-solves the national line, whose problem kernel hashes it. Both now say eight never re-solve on any line and diagnostics re-solves only the national line. Two comments beside the full gate nodes said an edit to one phase's own code re-keys only that phase, or never the preflight and the solve; they now say key chaining still re-keys them where a node upstream hashes the module. - Nit 2. gate_scopes' docstring, calibration_run's comment above its gate_scopes import, the docstring of test_an_edit_to_calibration_run_leaves_the_preflight_and_the_solve and docs/uk-full-build-graph.md now say what the changelog says: the move keeps the runner out of the full gate kernels' own hashes and the preflight's own code, so in the in-graph build an edit to the runner re-keys only the calibrated gates and what follows them, while on a bound spine the checkpoint, whose kernel hashes the runner, still re-keys the preflight and the solve. The doc also says the spine and local scopes and the scoped manifest are defined in gate_scopes and re-exported by calibration_run. - The changelog's gate battery fragment records the module, the check, the new refusal and source_hash leaving the core names, and counts seven refusal-only checks. Checked by simulated edits (a comment line through the module's __file__, the phase-code caches cleared), at the parent and here alike, counted as in-graph build / local line on a bound spine / national line: each of the eight re-keys only uk.full.gates.calibrated where it re-keys anything (1 of 113, 1 of 33, and 1 or 0 of 6); uk_runtime.diagnostics re-keys 1, 1 and 4 of 6 (the national problem, the dense solve, the calibrated population and the gates); calibration_run 1 of 113, 33 of 33 and 6 of 6. A static reach of the 38 terminal-only modules splits them the same way at the parent and here: eight re-solve on no line, one on the national line only, two on a bound spine only, seven in the in-graph build only, three in the in-graph build and on the local line, and seventeen on every line. What moves: node keys only, plus the spine gates' new refusal. No value, declaration or generated file moves, and neither the solve nor the country fingerprint does. - 72 of the 99 UK kernel hashes move: exactly those that hash graph_evidence.py (65), graph_terminal.py (8), gate_scopes.py (5) or calibration_run.py (2). No kernel's set of hashed modules changes, and none hashes gate_phase_codes. The other 27 keep theirs: the four atomic geography kernels, the claim and both identity kernels, the calibration and size kernels, the full target kernels, the geography, property and private rent kernels, and the national targets, problem and readback. No transport kernel hashes these files, and none moves. - Of the phase codes only terminal_code moves, since it hashes calibration_run.py; assembled_code, transferred_code and preflight_code keep their values. - So every node of every UK graph re-keys: the hermetic and gated spine (79 and 81), the in-graph build (113), the local line on a bound spine (33) and the national line (6). The bound spine kernel hashes gate_scopes.py and calibration_run.py, whose texts changed, so both lines on a bound spine re-key from their root. - After this, an edit to a phase's list re-keys that phase's node and what follows it, and nothing else. One module added to the transferred phase's list, simulated: at the parent, in graph_evidence.py, 65 kernel hashes move and 79 of 79, 81 of 81, 113 of 113, 32 of 33 and 0 of 6 nodes re-key (hermetic spine, gated spine, in-graph build, local line on a bound spine, national line); here, in gate_phase_codes.py, no kernel hash moves and 0, 1 (the transferred gate), 33 (that gate and every node after it), 0 and 0 re-key. A comment edit to gate_phase_codes.py re-keys nothing. - uk_spine.json and the H2 fixture hold stage contract hashes, and no fixture holds a gate node, so neither regenerates. source_stages.json and the coverage manifest are current. Tests (engine-free; code facts and synthetic values): test_uk_graph, test_uk_graph_evidence, test_uk_graph_terminal, test_uk_target_compile_modules and the shared rule and transport rule tests pass (168 tests), and 15 related files pass (655 tests). Through the engine slot, test_uk_full_graph_admission passes. 15 mutations on a scratch copy are caught: either gate kernel reading its table beside the check, or the full gates reading theirs in its place (the rule finds gate_phase_codes unhashed in both phases' walks); the check writing into its params or returning its code; the spine gates dropping the check, running it after their first artifact read or on the full battery's table, each caught both by the static test and by the spine refusal test; the full gates on the spine battery's table; a gate kernel hashing the module; graph_evidence re-exporting a table; and a phase's list gaining a module its gates do not compute through. Ruff is clean on the 10 edited Python files, and ci_test_plan verify is ok. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-helper-bindings.changed.md | 11 +- docs/uk-full-build-graph.md | 2 +- .../build/uk_runtime/calibration_run.py | 11 +- .../build/uk_runtime/gate_phase_codes.py | 345 ++++++++++++++++++ .../microcosm/build/uk_runtime/gate_scopes.py | 13 +- .../build/uk_runtime/graph_evidence.py | 150 ++------ .../build/uk_runtime/graph_terminal.py | 221 ++--------- .../engine/uk/test_uk_full_graph_admission.py | 2 +- .../tests/engine_free/uk/test_uk_graph.py | 168 +++++++-- .../engine_free/uk/test_uk_graph_evidence.py | 91 ++++- .../engine_free/uk/test_uk_graph_terminal.py | 35 +- .../uk/test_uk_target_compile_modules.py | 28 +- 12 files changed, 681 insertions(+), 396 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 50150467e2..341340eda4 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,7 +1,7 @@ Every UK spine stage now hashes each first-party module its transform computes through, and each stage that calls the engine also each one the injected engine computes through for it. Every other UK graph kernel hashes each one its own code computes through, the target and problem kernels among them, and each phase of a gate battery each one its own gates compute through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. - **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage, and for a stage that calls the engine also at the engine the builds inject (see The engine), and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports, module attributes and `type` aliases, which read as assignments of their values. So do the names `microcosm.graph` exports lazily through its module `__getattr__`: the walk resolves each to the runtime module that defines it, from the table that `__getattr__` reads, and a test checks that every first-party module with a module `__getattr__` is listed. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: - - six listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels); + - seven listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels), and one is the gate kernels' stale phase-code check (see Gate batteries); - the stage's own declaration (`country_spec`, `source_manifest` and the spec engine), whose content the node key already hashes as the stage contract; - the listed core modules every node runs through (see Scope); - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; two such names are listed. @@ -41,7 +41,7 @@ Every UK spine stage now hashes each first-party module its transform computes t - `UK_GATE_REGISTRY`, which the bound spine reaches only for its checkpoint's gate digests. The gate batteries and the national gates evaluate through it, and the rule holds them past it (see Gate batteries); - `microcosm.graph.codecs`' `load_source`, which the walk now sees through `microcosm.graph`'s lazy exports. The create kernel calls it only where no root transform is bound, the hermetic registry's data-only path, which loads the H2 fixture's captured root tables. The spine build installs the root's transform and H2 parity binds it too, so neither takes that path (checked). Binding the codecs, and the content store they name, would re-key the spine root, and with it every spine node and every node downstream of the spine, on edits to code no build runs. As María ruled on 9 Nov, it is a stop instead, and a test checks that no spine kernel and no kernel of the national line hashes the codecs. What the codecs decode is held by `microcosm-graph`'s codec tests. - The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. Another holds the executor's and key modules' definitions that the kernels' own code names to a listed table, each with a reason: the dtype tokens' map, which the executor casts every node's owned cells through anyway, two artifact declaration records, the capabilities record and its five enums, the kernel context and result records, and `source_hash`, through which the full gates' run recomputes its phase's own code only to refuse a stale one. The kernels name six of them through `microcosm.graph`'s lazy exports, which the walk now follows: the capabilities record, three of its enums, and the context and result records. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: + The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. Another holds the executor's and key modules' definitions that the kernels' own code names to a listed table, each with a reason: the dtype tokens' map, which the executor casts every node's owned cells through anyway, two artifact declaration records, the capabilities record and its five enums, and the kernel context and result records. The kernels name six of them through `microcosm.graph`'s lazy exports, which the walk now follows: the capabilities record, three of its enums, and the context and result records. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: - the seven dense population kernels' shared hash: the engine adapter, the canonical encoder and the weight-update receipt; - the geography identity: `household_lineage`, `source_lineage`, `microcosm.frame.concepts` and the canonical encoder; - property value: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `tenure_constants`, the keyed uniforms and the canonical encoder; @@ -59,9 +59,10 @@ Every UK spine stage now hashes each first-party module its transform computes t - the national readback: `artifact_files`, `trace`, `frame.materialize`, the engine adapter, `geography_ladder`, `graph_population`, `hmrc_uprating` and the canonical encoder. The LAD23 names' pin stays listed, now for every kernel that reads it. The geographic clone index columns' name (`rowwise_dataset.ladder_clone_index_column`) is listed for the calibrated and national gates, whose support-family fold reads it: only the dense expand writes those columns, and it hashes `rowwise_dataset`; the national frame has no ladder, and the fold keys on the column only where the table carries it. It is listed for the dense measures and the national problem too (see Target and problem kernels). `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside implementation hashes: the national gates' and readback's, and for three of them the national problem's. -- **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Where a phase's gates read the engine their kernel hands them, the walk also follows the methods they call on it, from the engine's class, as for an engine stage (see The engine): the spine gates' rules engine is the spine build's `PolicyEngineUKEngine`, and the full gates' engine is the driver's coverage engine. Another test checks each part of that model against the code, the engines among them. So the assembled gate's own key, which the spine stages after it chain on, carries nothing only the transferred phase computes through, and the preflight's, which the dense solve chains on, nothing only the terminal phase does. Key chaining still re-keys them wherever a node upstream hashes the edited module. Of the 17 modules only the transferred phase computes through, stages at or before `frs_brma`, the assembled phase's last, hash `stochastic_assignment` and `take_up_contract`, so an edit to either re-keys the assembled gate; no stage before it hashes the other 15. Of the 38 only the terminal phase computes through, most re-solve too, through the spine stages, the target and problem kernels, the population kernels or the dense solve, which hash them as well. Nine never re-solve, on a bound spine or in the in-graph build: `high_value_home_gates`, `uk_runtime.diagnostics`, `microcosm.diagnostics`' catalogs and schema, `calibrate.hierarchy_labels`, the target loss attribution, `input_mass`, `cgt_projection` and `source_runtime`, so a disclosure fix between two builds that must reuse the solve still belongs in `high_value_home_gates`. `calibration_run` and `release_certification` re-solve only on a bound spine, whose kernel hashes both, and seven modules that only the spine's nodes hash before the solve re-solve only in the in-graph build. The preflight's own code, the shared ledger's parity report, is also hashed by the full build's target compilation, selection, measures and problem, by the national line's targets, problem and gates and, in the in-graph build, by the `etb_services` and `lcfs_consumption` stages and the transferred gate, so an edit to it re-keys those too. - - The spine gates' hash binds the 12 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants`, `calibrate.geography_constants` and the engine adapter, through whose `PolicyEngineUKEngine.enum_domain` each phase's enum gates resolve a domain no artifact supplies, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. As before, the spine gate kernel does not check its phase's code. +- **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Where a phase's gates read the engine their kernel hands them, the walk also follows the methods they call on it, from the engine's class, as for an engine stage (see The engine): the spine gates' rules engine is the spine build's `PolicyEngineUKEngine`, and the full gates' engine is the driver's coverage engine. Another test checks each part of that model against the code, the engines among them. So the assembled gate's own key, which the spine stages after it chain on, carries nothing only the transferred phase computes through, and the preflight's, which the dense solve chains on, nothing only the terminal phase does. Key chaining still re-keys them wherever a node upstream hashes the edited module. Of the 17 modules only the transferred phase computes through, stages at or before `frs_brma`, the assembled phase's last, hash `stochastic_assignment` and `take_up_contract`, so an edit to either re-keys the assembled gate; no stage before it hashes the other 15. Of the 38 only the terminal phase computes through, most re-solve too, through the spine stages, the target and problem kernels, the population kernels or the dense solve, which hash them as well. Eight never re-solve, on any line: `high_value_home_gates`, `microcosm.diagnostics`' catalogs and schema, `calibrate.hierarchy_labels`, the target loss attribution, `input_mass`, `cgt_projection` and `source_runtime`, so a disclosure fix between two builds that must reuse the solve still belongs in `high_value_home_gates`. `uk_runtime.diagnostics` re-solves only the national line, whose problem kernel hashes it. `calibration_run` and `release_certification` re-solve only on a bound spine, whose kernel hashes both, and seven modules that only the spine's nodes hash before the solve re-solve only in the in-graph build. The preflight's own code, the shared ledger's parity report, is also hashed by the full build's target compilation, selection, measures and problem, by the national line's targets, problem and gates and, in the in-graph build, by the `etb_services` and `lcfs_consumption` stages and the transferred gate, so an edit to it re-keys those too. + - The spine gates' hash binds the 12 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants`, `calibrate.geography_constants` and the engine adapter, through whose `PolicyEngineUKEngine.enum_domain` each phase's enum gates resolve a domain no artifact supplies, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. The spine gate kernel refuses a stale or missing `_code`, as the full gates do (see The phase codes, below). - The full gates' hash binds the 26 modules both phases compute through. It loses `local_rowwise`, `weighted_integrity` and `diagnostics`, which only the terminal phase computes through. It binds the spine and local gate scopes and the scoped gate manifest, which the preflight computes through, in a new `gate_scopes` module, not in `calibration_run`, the national calibration runner they lived in. They moved there unchanged, and `calibration_run` re-exports them, as microcosm#1492 moved the target kernels' two functions into `ledger_surface`, so in the in-graph build an edit elsewhere in the runner re-keys neither the preflight nor the solve. The bound spine, the national gates and the certification, whose code reaches them too, hash `gate_scopes` as well. The calibrated gates' `terminal_code` binds `calibration_run` itself, for the admin totals and CGT projection artifact their gate context is built with. The preflight's release input coverage gate computes through `stage_evidence`, `parity_reference`, `release_identity` and `release_input_coverage`, which its key used to miss, so on a bound spine an edit to any of them now re-keys the preflight and re-solves; in the in-graph build so does one to `parity_reference` or `release_identity`, while the other two already re-solved through the spine stages that hash them. The preflight node carries a new `preflight_code`: the shared ledger's registry parity, behind the ledger compile parity gates only the preflight declares. The calibrated node's `terminal_code` now hashes 38 modules. `high_value_homes` and `calibrate.geography_constants` move into the shared hash, since the preflight computes through them too; the target kernels hash both, so an edit to either already re-solved. The kernel refuses a stale or missing `preflight_code`, as it refuses a stale `terminal_code`. + - The phase codes. As María ruled on 9 Nov (late), the four functions that compute them and their two tables live in a small module of their own, `uk_runtime.gate_phase_codes`, unchanged in what they hash, with one stale-code check, `require_current_phase_code`, which both batteries' kernels call before they read anything of their node but its parameters. In `graph_evidence` and `graph_terminal`, which define the gate kernels and which every spine stage kernel, the dense population kernels and the export kernels hash too, an edit to a phase's list of modules, which the battery test demands whenever a phase's evaluators gain one, re-keyed every node whose kernel hashes either file: the whole spine, or the preflight and the solve. Now only the graph construction, which sets each node's `_code`, and the check read the module, and no kernel hashes it: the check is listed refusal-only, since it reads its parameters and the phase's code and refuses, writing nothing, and a phase's own code reaches its key through the parameter, which `node_key` hashes. So an edit to a phase's list moves only that phase's parameter, which re-keys that phase's node and what follows it. `source_hash` leaves the kernels' listed core names: only the full gates' run named it, through the phase-code table, which only the check reads now. The spine gates' run now refuses a stale or missing `_code` too, a new refusal, so a graph declaration replayed from another process cannot key a spine gate on stale phase code. Tests check that no UK kernel's key covers the module, that both gate kernels call the check first, on their own battery's table, and that each spine phase refuses a stale and a missing code and passes its node's own. - The national gates evaluate whichever gates their node's scope names, so their hash binds everything the gate registry's bindings compute through. That is 50 modules, which `battery_bindings.UK_GATE_REGISTRY_MODULES` lists and a test holds exactly to the registry's derived reach. Their stop at the registry, the one held kernel stop, goes. - **Target and problem kernels.** As María ruled on 9 Nov, the derived rule replaces microcosm#1497's per-kernel import record for the full build's target compilation, selection, measures and problem and the national targets and problem, which that record held until now. No kernel stays held. - `test_uk_target_compile_modules` loses the record (`UNCLASSIFIED_IMPORTS`), its kernel groups and the check that read them. It keeps the shared list's closure check, the tests of what the kernels hash and its hash inputs, the seven above among them. A new check covers what the derived rule does not: each hash input is still imported where listed, and a kernel hashes one only where another module it hashes imports it too. So no kernel hashes the compile list or the solve fingerprint's module, and an edit to either re-keys none. @@ -70,4 +71,4 @@ Every UK spine stage now hashes each first-party module its transform computes t - The measures and the national problem also bind `frs_spine` for one name, the annuity and trust column. The resolver reads that column where the frame carries it and reads zeros where it does not, so on the national line's bound spine a renamed constant would change the problem without re-keying it. The clone index columns' name stays listed for them: the dense measures read it only to split their frame into the engine blocks they are asked for, and refuse a frame without it, and their population is the expand's; the national problem resolves its measures in one block, which never reads it. A test checks these facts. - No kernel drops a module it hashed before, though each still hashes some modules its walk does not reach. The full kernels' shared hash still binds the whole compile list and the modules beside it in all four, so the selection's key covers 51 modules while its code computes through 6. - `transport/compile_modules.py`'s reason for leaving the graph runtime off the transport compile list said that no kernel in the repository hashes a `microcosm.graph` module, which was untrue. It now says what the listed modules use those modules for, that the compile kernels hash none of them, and that other kernels hash the canonical encoder, each UK graph kernel whose code computes through it among them. No kernel hashes that file, so the edit re-keys nothing. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. The target and problem kernels' keys move once with their new bindings. `graph_targets.py` and `graph_national.py`, which define them, changed with them, and the preflight and calibrated gates, the rotated holdout and the national gates and readback each hash one or both of those files too. So in the full build the target compilation, selection, measures and problem, the pool checkpoint (`uk.full.pool`), the preflight, the rotated holdout, and the dense solve and every node after it re-key, and on the national line every node after its bound spine root; no spine node and no transport node re-keys with them. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. The target and problem kernels' keys move once with their new bindings. `graph_targets.py` and `graph_national.py`, which define them, changed with them, and the preflight and calibrated gates, the rotated holdout and the national gates and readback each hash one or both of those files too. So in the full build the target compilation, selection, measures and problem, the pool checkpoint (`uk.full.pool`), the preflight, the rotated holdout, and the dense solve and every node after it re-key, and on the national line every node after its bound spine root; no spine node and no transport node re-keys with them. The phase codes' move re-keys once more every node whose kernel hashes `graph_evidence.py`, `graph_terminal.py`, `gate_scopes.py` or `calibration_run.py`, which is every node of every UK graph, the bound spine of the local and national lines included, whose kernel hashes the last two; of the phase codes only `terminal_code` moves, since it hashes `calibration_run`. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index bd1c211695..37701ef556 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -65,7 +65,7 @@ For smaller outputs, add `--dataset-households 150000` to use the common informe ## The national role -`--release-role national` builds the certified national line on the same graph. The driver parses and validates it with the same posture-aware validator (the national role refuses the dense role's arguments, `--release-candidate` among them, and requires a bound `--input-h5` with `--input-sha256`; a `--spine-request` is refused because the national line solves a pinned checkpoint as bound) and composes the national graph (`uk_runtime.graph_national`): `uk.full.spine_checkpoint` admits the bound checkpoint, `uk.full.national_targets` compiles the national register from the pinned Chronicle artifact (measure exclusions, band edges, the frozen-register check, the feed pin with `--allow-unpinned-feed`; no ladder, no local surface), `uk.full.national_problem` refuses non-finite inputs, resolves the engine measures on the bound frame, materialises the register and encodes one ordered problem with the national doctrine's `family_equal` loss weights, mass reason and bounds in its bindings, `uk.full.dense` and `uk.full.calibrated` are the shared calibration nodes under the doctrine's epochs, learning rate and seed (the dense node's admission is the bound spine's provenance artifact rather than a preflight battery, because the national line runs no pre-solve battery: its source and reference gates are the release-cut certifier's), and `uk.full.gates.calibrated` evaluates the seven `UK_CALIBRATION_GATE_SCOPE` gates on the calibrated population. The driver then materialises the seam-shaped evidence from the stored artifacts: the diagnostics through the seam writer with the build block (the digest measured from the file), the target-support sidecars, the calibration-seam battery replayed from the stored phase report and signed (`GateBatteryRun.record_phase`), `microcosm_uk_2024_25.h5`, a graph readback of the written H5 (`uk.full.national.readback`), the schema-1 `build_record.json` that `tools/certify_uk_release_cut.py` certifies, the frozen `national_target_registry.json` and `national_contract_registry.json`, and the schema-4 `rowwise_candidate_manifest.json` (`uk_runtime.national_role`). The Logbook row is on the seam's `uk-frs-calibration` pipeline with the seam's attempt id; staging telemetry, the staged bundle and the end-of-build comparison with the reference (the current national build, microcosm#1095) follow the dense role's switches. A blocked battery leaves the report on disk (`blocked_at_phase: "terminal"`), writes no H5 and records a failed row. The [national calibration runbook](uk-national-calibration-runbook-623.md) and the [national release assembly runbook](uk-national-release-assembly-runbook-806.md) describe the outputs. The in-process calibration seam (`run_uk_calibration`, `UKNationalCalibrationStage`) is retired; its contracts (gate scopes, sidecar authentication, provenance blocks, scoped signing) stay in `uk_runtime.calibration_run`. Parity with the seam is a solved question of construction (the same register, the same matrix row for row, the same solver under the same doctrine and seed, the same mass record) checked on the licensed spine. +`--release-role national` builds the certified national line on the same graph. The driver parses and validates it with the same posture-aware validator (the national role refuses the dense role's arguments, `--release-candidate` among them, and requires a bound `--input-h5` with `--input-sha256`; a `--spine-request` is refused because the national line solves a pinned checkpoint as bound) and composes the national graph (`uk_runtime.graph_national`): `uk.full.spine_checkpoint` admits the bound checkpoint, `uk.full.national_targets` compiles the national register from the pinned Chronicle artifact (measure exclusions, band edges, the frozen-register check, the feed pin with `--allow-unpinned-feed`; no ladder, no local surface), `uk.full.national_problem` refuses non-finite inputs, resolves the engine measures on the bound frame, materialises the register and encodes one ordered problem with the national doctrine's `family_equal` loss weights, mass reason and bounds in its bindings, `uk.full.dense` and `uk.full.calibrated` are the shared calibration nodes under the doctrine's epochs, learning rate and seed (the dense node's admission is the bound spine's provenance artifact rather than a preflight battery, because the national line runs no pre-solve battery: its source and reference gates are the release-cut certifier's), and `uk.full.gates.calibrated` evaluates the seven `UK_CALIBRATION_GATE_SCOPE` gates on the calibrated population. The driver then materialises the seam-shaped evidence from the stored artifacts: the diagnostics through the seam writer with the build block (the digest measured from the file), the target-support sidecars, the calibration-seam battery replayed from the stored phase report and signed (`GateBatteryRun.record_phase`), `microcosm_uk_2024_25.h5`, a graph readback of the written H5 (`uk.full.national.readback`), the schema-1 `build_record.json` that `tools/certify_uk_release_cut.py` certifies, the frozen `national_target_registry.json` and `national_contract_registry.json`, and the schema-4 `rowwise_candidate_manifest.json` (`uk_runtime.national_role`). The Logbook row is on the seam's `uk-frs-calibration` pipeline with the seam's attempt id; staging telemetry, the staged bundle and the end-of-build comparison with the reference (the current national build, microcosm#1095) follow the dense role's switches. A blocked battery leaves the report on disk (`blocked_at_phase: "terminal"`), writes no H5 and records a failed row. The [national calibration runbook](uk-national-calibration-runbook-623.md) and the [national release assembly runbook](uk-national-release-assembly-runbook-806.md) describe the outputs. The in-process calibration seam (`run_uk_calibration`, `UKNationalCalibrationStage`) is retired; its contracts (gate scopes, sidecar authentication, provenance blocks, scoped signing) stay in `uk_runtime.calibration_run`, except the spine and local gate scopes and the scoped gate manifest, which are defined in `uk_runtime.gate_scopes` and re-exported by `calibration_run`. The full gate kernels hash `gate_scopes` in place of the runner, and the preflight's own code leaves the runner out, so in the in-graph build an edit to the runner re-keys only the calibrated gates and what follows them; on a bound spine the checkpoint, whose kernel hashes the runner, still re-keys the preflight and the solve, and on the national line every node. Parity with the seam is a solved question of construction (the same register, the same matrix row for row, the same solver under the same doctrine and seed, the same mass record) checked on the licensed spine. ## Graph owners and shared contracts diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 96629d011b..f8f8495620 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -49,10 +49,13 @@ UK_NHS_SPENDING_COMPONENT_COLUMNS, ) -# Moved to gate_scopes, which the full build's gate battery hashes, so an -# edit elsewhere in this module no longer re-keys its preflight or the solve -# keyed on it (microcosm#931, as microcosm#1492 moved the two functions -# below). Kept importable from here for this module's other callers. +# Moved to gate_scopes, which the full build's gate battery hashes, so the +# full gate kernels' own hashes and the preflight's own code leave this +# module out (microcosm#931, as microcosm#1492 moved the two functions +# below): in the in-graph build an edit elsewhere in it re-keys only the +# calibrated gates and what follows them. On a bound spine the checkpoint, +# whose kernel hashes this module, still re-keys the preflight and the solve. +# Kept importable from here for this module's other callers. from microcosm.build.uk_runtime.gate_scopes import ( UK_LOCAL_GATE_SCOPE, UK_SPINE_GATE_SCOPE, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py new file mode 100644 index 0000000000..6688210afd --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py @@ -0,0 +1,345 @@ +"""Each UK gate battery phase's own code, and the check that refuses a stale one. + +A gate battery's phases run one kernel's code, and that kernel's hash is the +code they share. What only one phase's gates compute through is that phase's +own code, a source hash its node carries as its ``_code`` parameter, +so no other phase's own key carries it (microcosm#931 ruling 6): the spine +gates' assembled and transferred phases (:data:`UK_SPINE_GATE_PHASE_CODE`) +and the full build's preflight and calibrated gates +(:data:`UK_FULL_GATE_PHASE_CODE`). + +The four functions and two tables lived in ``graph_evidence`` and +``graph_terminal``, which define the gate kernels and which every spine +stage kernel, the dense population kernels and the export kernels hash too, +so an edit to a phase's list of modules re-keyed every node whose kernel +hashes either file. They moved here unchanged in what they hash, with the +one check both batteries call (microcosm#931 ruling 3a). Only the graph +construction, which sets each node's parameter, and that check read this +module. The check is listed refusal-only in test_uk_graph, and no kernel +hashes this module: a phase's own code reaches its key through the +parameter, which node_key hashes, and both gate kernels' runs call +:func:`require_current_phase_code` before they evaluate anything, so a node +whose parameter differs from the code this process computes is refused +(microcosm#931 ruling 3b, for the spine gates). +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from functools import cache +from types import MappingProxyType + +from microcosm.graph import source_hash + + +@cache +def uk_spine_assembled_only_code_hash() -> str: + """The source hash of the code only the assembled spine phase computes + through: the ONS household type domain its enum gate checks + (``frs_relationships``), which no transferred-phase gate reads. + + So the transferred phase's own key leaves it out, though an edit to it + still re-keys the transferred gate, through the ``frs_relationships`` + stage, whose module it is, and every stage after it. Hashed once per + process, as for the calibrated gates' own code + (:func:`uk_terminal_only_code_hash`). + """ + from . import frs_relationships + + return source_hash(frs_relationships) + + +@cache +def uk_spine_transferred_only_code_hash() -> str: + """The source hash of the code only the transferred spine phase computes + through, which the assembled gate's key leaves out. + + That is what the stage-health checks only the transferred phase declares + compute through: the energy rake's published level and NEED margins + (``energy_pricing``); the bus travel, bus pricing, bus support pricing, + consumption basis and gambling checks' published facts and pricing + (``nts_bus_travel``, ``bus_use_incidence``, ``bus_fare_pricing``, + ``bus_support_pricing``, ``etb_services``, ``lcfs_consumption``, + ``donor_uprating``, ``gambling_participation``, the shared ledger's + calendar-year weights, and the vendored-fact reader with the chronicle + feed and its schema epochs); the annual income spells check + (``frs_annual_income``); the apprentice draw's take-up contract + (``take_up_contract``); and the latent-attribute checks' binomial band + (``stochastic_assignment``). Its + enum gate also checks the CGT asset type domain (``cgt_asset_type``). + + The assembled gate's own key, on which every later spine stage's chains, + therefore carries none of them (the re-review of microcosm#1420, E; + microcosm#931 ruling 6). An edit to one still re-keys the assembled gate + wherever a stage before it hashes the module: stages at or before + ``frs_brma``, the assembled phase's last, hash ``stochastic_assignment`` + and ``take_up_contract``, so an edit to either re-keys every spine node + after them, the assembled gate included. No stage before that gate hashes + the other fifteen, so an edit to one of those re-keys the stages that + hash it, those after them and the transferred gate, never the assembled + gate. Hashed once per process, as the assembled phase's own code is. + """ + from .. import chronicle_epoch, stochastic_assignment + from .. import ledger_targets as shared_ledger_targets + from . import ( + bus_fare_pricing, + bus_support_pricing, + bus_use_incidence, + cgt_asset_type, + chronicle_feed, + donor_uprating, + energy_pricing, + etb_services, + frs_annual_income, + gambling_participation, + lcfs_consumption, + ledger_fact_vendoring, + nts_bus_travel, + take_up_contract, + ) + + return source_hash( + energy_pricing, + lcfs_consumption, + bus_fare_pricing, + bus_support_pricing, + bus_use_incidence, + nts_bus_travel, + etb_services, + gambling_participation, + donor_uprating, + frs_annual_income, + take_up_contract, + cgt_asset_type, + stochastic_assignment, + chronicle_feed, + chronicle_epoch, + ledger_fact_vendoring, + shared_ledger_targets, + ) + + +#: Each spine gate phase's own code: the hash of what only that phase's gates +#: compute through, which its node carries as its ``_code`` parameter +#: and the spine gate kernel checks before it evaluates anything +#: (:func:`require_current_phase_code`), so no other phase's own key carries +#: it. An edit there still re-keys the other phase wherever a node upstream of +#: it hashes the module, as each function's docstring says. The code both +#: phases compute through is the gate kernel's own hash (microcosm#931 ruling +#: 6); test_uk_graph derives each phase's share from the gates it declares. +UK_SPINE_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( + { + "assembled": uk_spine_assembled_only_code_hash, + "transferred": uk_spine_transferred_only_code_hash, + } +) + + +@cache +def uk_terminal_only_code_hash() -> str: + """The source hash of the code only the calibrated gates compute through. + + The source preflight runs the gate kernel's code and the dense solve keys + on the preflight, so the code only the terminal phase computes through + keys the calibrated node through its ``terminal_code`` parameter, not the + kernel's hash (microcosm#1306 review; microcosm#931 ruling 6), and the + preflight's own key, and with it the solve's, carries none of it. An edit + to one of these modules re-keys the calibrated gates and what follows + them, and re-solves too wherever a node upstream of the solve hashes the + module, as a spine stage, a target or problem kernel, a population kernel + or the dense solve itself does for most of them. Eight never re-solve, on + any line: ``high_value_home_gates``, ``microcosm.diagnostics``' catalogs + and schema, ``calibrate.hierarchy_labels``, the target loss attribution, + ``input_mass``, ``cgt_projection`` and ``source_runtime``. + ``uk_runtime.diagnostics`` re-solves only the national line, whose problem + kernel hashes it. ``calibration_run`` and ``release_certification`` + re-solve only on a bound spine, whose kernel hashes both, and seven that + upstream of the solve only the spine's stage and gate nodes hash re-solve + only in the in-graph build (``donor_uprating``, ``etb_services``, + ``etb_vat``, ``frs_take_up``, ``hmrc_capital_gains``, + ``take_up_contract``, ``uc_capital_coherence``). That is the terminal + branch of the full gate kernel's run, and what the terminal gates compute + through beyond the preflight's: + + - the high-value homes evidence, on the calibrated pool alone + (``high_value_home_gates``, with the disclosure floor it reads from + ``receipt_counts`` and the tenure categories behind its owner mask, + ``tenure_constants``). ``high_value_homes`` itself is in the kernel's + hash, since the preflight's target-surface evidence reads its metric + names, and the target kernels hash it too, so an edit to it re-solves: + a disclosure fix between two builds that must reuse the solve belongs + in ``high_value_home_gates`` alone. ``calibrate.geography_constants``, + which the preflight's ledger parity reads, is in the kernel's hash too; + - the calibration diagnostics the calibrated node publishes as schema 9: + ``uk_runtime.diagnostics``, with its consumption drift + (``domestic_energy_targets``, ``energy_pricing``, ``etb_vat``, + ``lcfs_consumption``) and label choices (``local_authority_input``, + ``calibrate.hierarchy_labels``); ``microcosm.calibrate``'s + diagnostics, target loss attribution and solve; and + ``microcosm.diagnostics``' catalogs and schema; + - the solved problem and result it decodes (``microcosm.calibrate``'s + artifacts, matrix and monetary binding), the frame it rebuilds + (``graph_population``), the area-support summary (``local_rowwise``), + the support floors, admin anchors and CGT projection it builds the + gate context with (``support_floor``, ``etb_services``, + ``cgt_projection``, ``donor_uprating``, and the runner's admin totals + and projection artifact, ``calibration_run``), the input-mass reference + (``weighted_integrity``) and the fit-weight records it rehydrates + (``release_certification``); + - the terminal gates' evaluators: the input-mass parity + (``input_mass``), the CGT projection fence's conditioning facts + (``hmrc_capital_gains``, read through the vendored-fact reader, the + chronicle feed and its schema epochs), the take-up signal + (``frs_take_up``, ``take_up_contract``, ``dfe_spring_term``), the + column implication's property shares (``uc_capital_coherence``), the + nonnegative columns (``source_runtime``) and the geography ladder + (``rowwise_geography``). + + Hashed once per process, as the code was imported: an edit to the files + while a build runs cannot fail the calibrated gates after the solve. + """ + import microcosm.calibrate._target_loss_attribution as target_loss_attribution + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.diagnostics as calibrate_diagnostics + import microcosm.calibrate.hierarchy_labels as hierarchy_labels + import microcosm.calibrate.matrix as calibrate_matrix + import microcosm.calibrate.monetary_binding as monetary_binding + import microcosm.calibrate.solve as calibrate_solve + import microcosm.diagnostics.catalogs as diagnostics_catalogs + import microcosm.diagnostics.schema as diagnostics_schema + + from .. import chronicle_epoch, input_mass + from . import ( + calibration_run, + cgt_projection, + chronicle_feed, + dfe_spring_term, + diagnostics, + domestic_energy_targets, + donor_uprating, + energy_pricing, + etb_services, + etb_vat, + frs_take_up, + graph_population, + high_value_home_gates, + hmrc_capital_gains, + lcfs_consumption, + ledger_fact_vendoring, + local_authority_input, + local_rowwise, + receipt_counts, + release_certification, + rowwise_geography, + source_runtime, + support_floor, + take_up_contract, + tenure_constants, + uc_capital_coherence, + weighted_integrity, + ) + + return source_hash( + high_value_home_gates, + diagnostics_catalogs, + diagnostics_schema, + hierarchy_labels, + diagnostics, + calibrate_diagnostics, + target_loss_attribution, + calibrate_artifacts, + calibrate_matrix, + monetary_binding, + calibrate_solve, + graph_population, + local_rowwise, + support_floor, + rowwise_geography, + release_certification, + weighted_integrity, + input_mass, + calibration_run, + etb_services, + etb_vat, + domestic_energy_targets, + energy_pricing, + lcfs_consumption, + cgt_projection, + donor_uprating, + hmrc_capital_gains, + frs_take_up, + take_up_contract, + dfe_spring_term, + receipt_counts, + uc_capital_coherence, + tenure_constants, + source_runtime, + local_authority_input, + ledger_fact_vendoring, + chronicle_feed, + chronicle_epoch, + ) + + +@cache +def uk_preflight_only_code_hash() -> str: + """The source hash of the code only the source preflight computes + through: the shared ledger's registry parity report, behind the ledger + compile parity gates only the preflight declares + (``microcosm.build.ledger_targets``). + + The preflight's node carries it as its ``preflight_code`` parameter, so + the calibrated gates' own key leaves it out (microcosm#931 ruling 6). An + edit to it re-keys the preflight, the solve after it and the calibrated + gates through them, and every other node that hashes it: the full build's + target compilation, selection, measures and problem, the national line's + targets, problem and gates, and in the in-graph build the + ``etb_services`` and ``lcfs_consumption`` stages and the transferred + spine gate, whose own code it is too. Hashed once per process, as the + calibrated gates' own code is. + """ + from .. import ledger_targets as shared_ledger_targets + + return source_hash(shared_ledger_targets) + + +#: Each full gate phase's own code: the hash of what only that phase computes +#: through, which its node carries as its ``_code`` parameter and the +#: kernel checks before it evaluates anything +#: (:func:`require_current_phase_code`). The code both phases compute through +#: is the gate kernels' shared hash (microcosm#931 ruling 6); test_uk_graph +#: derives each phase's share from the gates it declares and the branch of the +#: kernel's run it takes. +UK_FULL_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( + { + "preflight": uk_preflight_only_code_hash, + "terminal": uk_terminal_only_code_hash, + } +) + + +def require_current_phase_code( + params: Mapping[str, object], phase: str, *, battery: str +) -> None: + """Refuse a gate node unless its ``_code`` parameter is its phase's + own code as this process computes it (microcosm#931 ruling 3). + + ``battery`` is ``"spine"`` or ``"full"``, the battery whose table holds the + phase's code (:data:`UK_SPINE_GATE_PHASE_CODE`, + :data:`UK_FULL_GATE_PHASE_CODE`); a phase its table leaves out computes + through nothing of its own, and its node must carry no parameter. The + graph construction sets the parameter and node_key hashes it, so a node + declared in another process, or replayed from a stored graph, with a stale + or missing parameter is refused before its kernel evaluates anything. The + check reads only its arguments and the phase's code, and writes nothing, + so it cannot change what its caller computes: test_uk_graph lists it as + refusal-only, and no gate kernel's hash binds this module. + """ + tables = {"spine": UK_SPINE_GATE_PHASE_CODE, "full": UK_FULL_GATE_PHASE_CODE} + if battery not in tables: + raise ValueError(f"No UK gate battery {battery!r} carries phase codes.") + code = tables[battery].get(phase) + if params.get(f"{phase}_code") != (None if code is None else code()): + raise ValueError( + f"UK {battery} gate {phase}-only code differs from its declared binding." + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py index f58e980c01..7a05e20c0f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py @@ -6,11 +6,14 @@ the local gates (``full_gates.classify_full_gate_outcomes``), and the dense solve keys on the source preflight. So the battery's shared hash binds this small module instead of all of ``calibration_run`` (microcosm#931), as the -target kernels hash ``ledger_surface`` (microcosm#1492), and an edit -elsewhere in the runner re-keys neither the preflight nor, in the in-graph -build, the solve keyed on it. The calibrated gates' own code still binds the -runner, whose admin totals and CGT projection artifact their gate context is -built with. ``calibration_run`` re-exports all three for its other callers, +target kernels hash ``ledger_surface`` (microcosm#1492). That keeps the +runner out of the full gate kernels' own hashes and the preflight's own +code. The calibrated gates' own code still binds the runner, whose admin +totals and CGT projection artifact their gate context is built with, so in +the in-graph build an edit elsewhere in the runner re-keys only the +calibrated gates and what follows them. On a bound spine the checkpoint, +whose kernel hashes the runner, still re-keys the preflight and the solve +keyed on it. ``calibration_run`` re-exports all three for its other callers, beside the calibration, national and local diagnostic scopes it keeps. """ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py index 3adf9d7660..c77f2e497b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -4,10 +4,8 @@ import hashlib import json -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import replace -from functools import cache -from types import MappingProxyType from microcosm.graph import ( ArtifactInput, @@ -176,108 +174,6 @@ def uk_spine_gate_artifacts(engine: object) -> dict[str, object]: } -@cache -def uk_spine_assembled_only_code_hash() -> str: - """The source hash of the code only the assembled spine phase computes - through: the ONS household type domain its enum gate checks - (``frs_relationships``), which no transferred-phase gate reads. - - So the transferred phase's own key leaves it out, though an edit to it - still re-keys the transferred gate, through the ``frs_relationships`` - stage, whose module it is, and every stage after it. Hashed once per - process, as for the calibrated gates' own code - (``graph_terminal.uk_terminal_only_code_hash``). - """ - from . import frs_relationships - - return source_hash(frs_relationships) - - -@cache -def uk_spine_transferred_only_code_hash() -> str: - """The source hash of the code only the transferred spine phase computes - through, which the assembled gate's key leaves out. - - That is what the stage-health checks only the transferred phase declares - compute through: the energy rake's published level and NEED margins - (``energy_pricing``); the bus travel, bus pricing, bus support pricing, - consumption basis and gambling checks' published facts and pricing - (``nts_bus_travel``, ``bus_use_incidence``, ``bus_fare_pricing``, - ``bus_support_pricing``, ``etb_services``, ``lcfs_consumption``, - ``donor_uprating``, ``gambling_participation``, the shared ledger's - calendar-year weights, and the vendored-fact reader with the chronicle - feed and its schema epochs); the annual income spells check - (``frs_annual_income``); the apprentice draw's take-up contract - (``take_up_contract``); and the latent-attribute checks' binomial band - (``stochastic_assignment``). Its - enum gate also checks the CGT asset type domain (``cgt_asset_type``). - - The assembled gate's own key, on which every later spine stage's chains, - therefore carries none of them (the re-review of microcosm#1420, E; - microcosm#931 ruling 6). An edit to one still re-keys the assembled gate - wherever a stage before it hashes the module: stages at or before - ``frs_brma``, the assembled phase's last, hash ``stochastic_assignment`` - and ``take_up_contract``, so an edit to either re-keys every spine node - after them, the assembled gate included. No stage before that gate hashes - the other fifteen, so an edit to one of those re-keys the stages that - hash it, those after them and the transferred gate, never the assembled - gate. Hashed once per process, as the assembled phase's own code is. - """ - from .. import chronicle_epoch, stochastic_assignment - from .. import ledger_targets as shared_ledger_targets - from . import ( - bus_fare_pricing, - bus_support_pricing, - bus_use_incidence, - cgt_asset_type, - chronicle_feed, - donor_uprating, - energy_pricing, - etb_services, - frs_annual_income, - gambling_participation, - lcfs_consumption, - ledger_fact_vendoring, - nts_bus_travel, - take_up_contract, - ) - - return source_hash( - energy_pricing, - lcfs_consumption, - bus_fare_pricing, - bus_support_pricing, - bus_use_incidence, - nts_bus_travel, - etb_services, - gambling_participation, - donor_uprating, - frs_annual_income, - take_up_contract, - cgt_asset_type, - stochastic_assignment, - chronicle_feed, - chronicle_epoch, - ledger_fact_vendoring, - shared_ledger_targets, - ) - - -#: Each spine gate phase's own code: the hash of what only that phase's gates -#: compute through, which its node carries as its ``_code`` parameter, -#: so no other phase's own key carries it. An edit there still re-keys the -#: other phase wherever a node upstream of it hashes the module, as each -#: function's docstring says. The code both phases compute through is the -#: gate kernel's own hash (microcosm#931 ruling 6); test_uk_graph derives each -#: phase's share from the gates it declares. -UK_SPINE_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( - { - "assembled": uk_spine_assembled_only_code_hash, - "transferred": uk_spine_transferred_only_code_hash, - } -) - - def load_spine_stage_artifacts( manifest, store, *, stage_names: Sequence[str] ) -> dict[str, dict[str, object]]: @@ -331,6 +227,7 @@ def add_uk_spine_gate_nodes( data-only fixture is not the population its facts describe, so population-fact gates never block such a run. """ + from .gate_phase_codes import UK_SPINE_GATE_PHASE_CODE from .graph import uk_spine_endpoint gates = uk_spine_gate_manifest(spec) @@ -452,13 +349,18 @@ class UKSpineGateKernel(KernelBase): ``synthetic_smoke`` mirrors ``GateBatteryRun(synthetic_smoke=...)`` — on a data-only fixture the population-fact gates are recorded but never block. Each phase also carries ``_code``, the hash of the code only its - own gates compute through (:data:`UK_SPINE_GATE_PHASE_CODE`), so the - assembled gate's key, on which every later spine stage's chains, carries - none of the transferred phase's (the re-review of microcosm#1420, E; - microcosm#931 ruling 6). An edit to a module only the transferred phase - computes through still re-keys the assembled gate where a stage before it - hashes the module, as for ``stochastic_assignment`` and - ``take_up_contract``. + own gates compute through + (:data:`.gate_phase_codes.UK_SPINE_GATE_PHASE_CODE`), so the assembled + gate's key, on which every later spine stage's chains, carries none of the + transferred phase's (the re-review of microcosm#1420, E; microcosm#931 + ruling 6). An edit to a module only the transferred phase computes through + still re-keys the assembled gate where a stage before it hashes the + module, as for ``stochastic_assignment`` and ``take_up_contract``. The + kernel refuses a stale or missing ``_code`` before it evaluates + anything, as the full gates do + (:func:`.gate_phase_codes.require_current_phase_code`; microcosm#931 + ruling 3b), so a graph declaration replayed from another process cannot + key a spine gate on stale phase code. """ ref = "uk.spine-gates@1" @@ -503,13 +405,14 @@ def implementation_hash(self) -> str: # and the engine the kernel is built with, through which each phase's # enum gates resolve a domain no artifact supplies (the engine # adapter's PolicyEngineUKEngine.enum_domain). What only one phase's - # gates compute through is that phase's own code, a node param - # (UK_SPINE_GATE_PHASE_CODE): the energy level, for one, re-keys the - # transferred gate and not the assembled gate every later spine stage - # waits on, since no stage before that gate hashes energy_pricing (the + # gates compute through is that phase's own code, a node param that + # run checks (UK_SPINE_GATE_PHASE_CODE, in gate_phase_codes, which this + # hash leaves out): the energy level, for one, re-keys the transferred + # gate and not the assembled gate every later spine stage waits on, + # since no stage before that gate hashes energy_pricing (the # re-review, E). A phase-only module a stage before the assembled gate # hashes still re-keys it by chaining - # (uk_spine_transferred_only_code_hash). + # (gate_phase_codes.uk_spine_transferred_only_code_hash). return hashlib.sha256( canonical_json( { @@ -535,8 +438,10 @@ def implementation_hash(self) -> str: def run(self, context: KernelContext) -> KernelResult: from microcosm.frame import Frame, MassChangeRecord + from .gate_phase_codes import require_current_phase_code from .graph_kernels import _minimal_frame + phase = str(context.params["phase"]) expected = json.dumps( gate_battery._gates_manifest_payload(self.gates), sort_keys=True ) @@ -547,6 +452,11 @@ def run(self, context: KernelContext) -> KernelResult: raise ValueError( "Spine gate binding differs from its declared manifest or engine." ) + # The phase's own code keys the node through its parameter, not this + # kernel's hash, so a node whose parameter is stale or missing is + # refused before anything is read or evaluated (microcosm#931 ruling + # 3b), as the full gates refuse theirs. + require_current_phase_code(context.params, phase, battery="spine") release_candidate = bool(context.params["release_candidate"]) synthetic_smoke = bool(context.params["synthetic_smoke"]) previous = context.artifacts.get("previous_gate") @@ -559,7 +469,7 @@ def run(self, context: KernelContext) -> KernelResult: synthetic_smoke=synthetic_smoke, ): report = gate_battery.GatePhaseReport( - phase=str(context.params["phase"]), + phase=phase, outcomes=tuple( gate_battery.GateOutcome( entry=entry, @@ -567,7 +477,7 @@ def run(self, context: KernelContext) -> KernelResult: reason="The assembled spine gate blocked this phase.", ) for entry in self.gates.gates - if entry.phase == context.params["phase"] + if entry.phase == phase ), ) return KernelResult( @@ -603,7 +513,7 @@ def run(self, context: KernelContext) -> KernelResult: ) report = evaluate_phase( self.gates, - str(context.params["phase"]), + phase, EvidenceContext( frame=frame, artifacts={**evidence, **uk_spine_gate_artifacts(self.engine)}, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 327721f37d..85cdcd706c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -12,12 +12,11 @@ import io import json import sys -from collections.abc import Callable, Mapping +from collections.abc import Mapping from dataclasses import asdict, replace -from functools import cache from importlib import metadata from pathlib import Path -from types import MappingProxyType, SimpleNamespace +from types import SimpleNamespace from typing import Any import numpy as np @@ -819,187 +818,6 @@ def _source_gate_evidence(context: KernelContext, engine): } -@cache -def uk_terminal_only_code_hash() -> str: - """The source hash of the code only the calibrated gates compute through. - - The source preflight runs the gate kernel's code and the dense solve keys - on the preflight, so the code only the terminal phase computes through - keys the calibrated node through its ``terminal_code`` parameter, not the - kernel's hash (microcosm#1306 review; microcosm#931 ruling 6), and the - preflight's own key, and with it the solve's, carries none of it. An edit - to one of these modules re-keys the calibrated gates and what follows - them, and re-solves too wherever a node upstream of the solve hashes the - module, as a spine stage, a target or problem kernel, a population kernel - or the dense solve itself does for most of them. Nine never re-solve, on - a bound spine or in the in-graph build: ``high_value_home_gates``, - ``uk_runtime.diagnostics``, ``microcosm.diagnostics``' catalogs and - schema, ``calibrate.hierarchy_labels``, the target loss attribution, - ``input_mass``, ``cgt_projection`` and ``source_runtime``. - ``calibration_run`` and ``release_certification`` re-solve only on a bound - spine, whose kernel hashes both, and seven that upstream of the solve only - the spine's stage and gate nodes hash re-solve only in the in-graph build - (``donor_uprating``, ``etb_services``, ``etb_vat``, ``frs_take_up``, - ``hmrc_capital_gains``, ``take_up_contract``, ``uc_capital_coherence``). - That is the terminal branch of the kernel's run, and what the terminal - gates compute through beyond the preflight's: - - - the high-value homes evidence, on the calibrated pool alone - (``high_value_home_gates``, with the disclosure floor it reads from - ``receipt_counts`` and the tenure categories behind its owner mask, - ``tenure_constants``). ``high_value_homes`` itself is in the kernel's - hash, since the preflight's target-surface evidence reads its metric - names, and the target kernels hash it too, so an edit to it re-solves: - a disclosure fix between two builds that must reuse the solve belongs - in ``high_value_home_gates`` alone. ``calibrate.geography_constants``, - which the preflight's ledger parity reads, is in the kernel's hash too; - - the calibration diagnostics the calibrated node publishes as schema 9: - ``uk_runtime.diagnostics``, with its consumption drift - (``domestic_energy_targets``, ``energy_pricing``, ``etb_vat``, - ``lcfs_consumption``) and label choices (``local_authority_input``, - ``calibrate.hierarchy_labels``); ``microcosm.calibrate``'s - diagnostics, target loss attribution and solve; and - ``microcosm.diagnostics``' catalogs and schema; - - the solved problem and result it decodes (``microcosm.calibrate``'s - artifacts, matrix and monetary binding), the frame it rebuilds - (``graph_population``), the area-support summary (``local_rowwise``), - the support floors, admin anchors and CGT projection it builds the - gate context with (``support_floor``, ``etb_services``, - ``cgt_projection``, ``donor_uprating``, and the runner's admin totals - and projection artifact, ``calibration_run``), the input-mass reference - (``weighted_integrity``) and the fit-weight records it rehydrates - (``release_certification``); - - the terminal gates' evaluators: the input-mass parity - (``input_mass``), the CGT projection fence's conditioning facts - (``hmrc_capital_gains``, read through the vendored-fact reader, the - chronicle feed and its schema epochs), the take-up signal - (``frs_take_up``, ``take_up_contract``, ``dfe_spring_term``), the - column implication's property shares (``uc_capital_coherence``), the - nonnegative columns (``source_runtime``) and the geography ladder - (``rowwise_geography``). - - Hashed once per process, as the code was imported: an edit to the files - while a build runs cannot fail the calibrated gates after the solve. - """ - import microcosm.calibrate._target_loss_attribution as target_loss_attribution - import microcosm.calibrate.artifacts as calibrate_artifacts - import microcosm.calibrate.diagnostics as calibrate_diagnostics - import microcosm.calibrate.hierarchy_labels as hierarchy_labels - import microcosm.calibrate.matrix as calibrate_matrix - import microcosm.calibrate.monetary_binding as monetary_binding - import microcosm.calibrate.solve as calibrate_solve - import microcosm.diagnostics.catalogs as diagnostics_catalogs - import microcosm.diagnostics.schema as diagnostics_schema - - from .. import chronicle_epoch, input_mass - from . import ( - calibration_run, - cgt_projection, - chronicle_feed, - dfe_spring_term, - diagnostics, - domestic_energy_targets, - donor_uprating, - energy_pricing, - etb_services, - etb_vat, - frs_take_up, - graph_population, - high_value_home_gates, - hmrc_capital_gains, - lcfs_consumption, - ledger_fact_vendoring, - local_authority_input, - local_rowwise, - receipt_counts, - release_certification, - rowwise_geography, - source_runtime, - support_floor, - take_up_contract, - tenure_constants, - uc_capital_coherence, - weighted_integrity, - ) - - return source_hash( - high_value_home_gates, - diagnostics_catalogs, - diagnostics_schema, - hierarchy_labels, - diagnostics, - calibrate_diagnostics, - target_loss_attribution, - calibrate_artifacts, - calibrate_matrix, - monetary_binding, - calibrate_solve, - graph_population, - local_rowwise, - support_floor, - rowwise_geography, - release_certification, - weighted_integrity, - input_mass, - calibration_run, - etb_services, - etb_vat, - domestic_energy_targets, - energy_pricing, - lcfs_consumption, - cgt_projection, - donor_uprating, - hmrc_capital_gains, - frs_take_up, - take_up_contract, - dfe_spring_term, - receipt_counts, - uc_capital_coherence, - tenure_constants, - source_runtime, - local_authority_input, - ledger_fact_vendoring, - chronicle_feed, - chronicle_epoch, - ) - - -@cache -def uk_preflight_only_code_hash() -> str: - """The source hash of the code only the source preflight computes - through: the shared ledger's registry parity report, behind the ledger - compile parity gates only the preflight declares - (``microcosm.build.ledger_targets``). - - The preflight's node carries it as its ``preflight_code`` parameter, so - the calibrated gates' own key leaves it out (microcosm#931 ruling 6). An - edit to it re-keys the preflight, the solve after it and the calibrated - gates through them, and every other node that hashes it: the full build's - target compilation, selection, measures and problem, the national line's - targets, problem and gates, and in the in-graph build the - ``etb_services`` and ``lcfs_consumption`` stages and the transferred - spine gate, whose own code it is too. Hashed once per process, as the - calibrated gates' own code is. - """ - from .. import ledger_targets as shared_ledger_targets - - return source_hash(shared_ledger_targets) - - -#: Each full gate phase's own code: the hash of what only that phase computes -#: through, which its node carries as its ``_code`` parameter and the -#: kernel checks before it evaluates anything. The code both phases compute -#: through is the gate kernels' shared hash (microcosm#931 ruling 6); -#: test_uk_graph derives each phase's share from the gates it declares and -#: the branch of the kernel's run it takes. -UK_FULL_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( - { - "preflight": uk_preflight_only_code_hash, - "terminal": uk_terminal_only_code_hash, - } -) - - class UKFullGateKernel(KernelBase): """The calibrated gate battery: every terminal gate, on the final weights. @@ -1008,7 +826,10 @@ class UKFullGateKernel(KernelBase): under :class:`UKFullPreflightGateKernel` (microcosm#1301). Both phases key on the code both compute through, this kernel's hash, and each on its own, which its node carries as ``_code`` and the kernel - checks (:data:`UK_FULL_GATE_PHASE_CODE`; microcosm#931 ruling 6). + checks before it evaluates anything + (:data:`.gate_phase_codes.UK_FULL_GATE_PHASE_CODE`, + :func:`.gate_phase_codes.require_current_phase_code`; microcosm#931 + rulings 3 and 6). """ ref = "uk.full-gates@1" @@ -1043,11 +864,12 @@ def implementation_hash(self) -> str: # target surfaces and their expected sets, the register loaders # (terminal_gates), and the spine and local gate scopes (gate_scopes, # not the runner that re-exports them). What only one phase computes - # through is that phase's own code (UK_FULL_GATE_PHASE_CODE), so the - # preflight's own key, and with it the dense solve's, carries none of - # the terminal phase's; an edit to one of those still re-solves - # wherever a node upstream of the solve hashes it - # (uk_terminal_only_code_hash). + # through is that phase's own code, a node param that run checks + # (UK_FULL_GATE_PHASE_CODE, in gate_phase_codes, which this hash + # leaves out), so the preflight's own key, and with it the dense + # solve's, carries none of the terminal phase's; an edit to one of + # those still re-solves wherever a node upstream of the solve hashes + # it (gate_phase_codes.uk_terminal_only_code_hash). import microcosm.calibrate.geography_constants as geography_constants import microcosm.calibrate.hierarchy as calibrate_hierarchy import microcosm.calibrate.registry as calibrate_registry @@ -1135,6 +957,7 @@ def run(self, context: KernelContext) -> KernelResult: uk_full_gate_manifest, uk_full_gate_scope_receipt, ) + from .gate_phase_codes import require_current_phase_code from .geography_ladder import load_uk_oa_ladder from .local_rowwise import uk_ladder_area_support_summary from .reference_surface import load_uk_reference_surface @@ -1145,10 +968,7 @@ def run(self, context: KernelContext) -> KernelResult: raise ValueError( "UK full gate engine identity differs from its declared binding." ) - if context.params.get(f"{phase}_code") != UK_FULL_GATE_PHASE_CODE[phase](): - raise ValueError( - f"UK full gate {phase}-only code differs from its declared binding." - ) + require_current_phase_code(context.params, phase, battery="full") selection = json.loads(context.artifacts["selection"].payload)["receipt"] gates = uk_full_gate_manifest(selection) from ..gate_battery import _gates_manifest_payload, _json_safe @@ -1425,6 +1245,7 @@ def append_uk_full_gate_nodes( from ..gate_battery import _gates_manifest_payload from ..stage_evidence import STAGE_EVIDENCE_TYPE from .full_gates import uk_full_gate_manifest + from .gate_phase_codes import UK_FULL_GATE_PHASE_CODE from .graph_evidence import SPINE_BUILD_STATE_TYPE, SPINE_GATE_REPORT_TYPE from .graph_targets import TARGET_SELECTION_TYPE, TARGET_SURFACE_TYPE @@ -1492,8 +1313,10 @@ def append_uk_full_gate_nodes( UKFullPreflightGateKernel.ref, population=input_population, # The preflight keys on its own code, as the calibrated node keys on - # its own, so an edit to either phase's alone re-keys only that phase - # and what follows it (microcosm#931 ruling 6). + # its own, so neither phase's own key carries the other's + # (microcosm#931 ruling 6). Key chaining still re-keys a phase wherever + # a node upstream of it hashes the edited module, as gate_phase_codes' + # docstrings say. params={ **params, "phase": "preflight", @@ -1543,8 +1366,10 @@ def append_uk_full_gate_nodes( population=calibration.population, inputs=population_slices(population_columns(graph, calibration.population)), sources=sources, - # Only the calibrated node keys on the code only it runs, so an edit to - # that code never re-keys the preflight and the solve after it. + # Only the calibrated node keys on the code only it runs, so the + # preflight's own key, and the solve's, leave it out; an edit to it + # still re-solves wherever a node upstream of the solve hashes the + # module (gate_phase_codes.uk_terminal_only_code_hash). params={ **gate_params, "phase": "terminal", diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py b/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py index b04e079f36..5f94a8d902 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py @@ -14,6 +14,7 @@ append_uk_full_certification_node, register_uk_full_certification_kernel, ) +from microcosm.build.uk_runtime.gate_phase_codes import uk_terminal_only_code_hash from microcosm.build.uk_runtime.graph_build import ( SPINE_PROVENANCE_TYPE, UKFullBuildConfig, @@ -29,7 +30,6 @@ decode_full_gate_report, register_uk_full_gate_kernels, register_uk_terminal_kernels, - uk_terminal_only_code_hash, ) from microcosm.graph import ( ArtifactInput, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 84d87df4a3..64054544c9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -1137,15 +1137,6 @@ def changed_module_bytes(path): "the executor fills, and every node's inputs pass through it, where " "H2 catches a changed value (graph.kernel's core reason)" ), - ("microcosm.graph.kernel", "source_hash"): ( - "hashes the source files of the modules it is given. The full gates' " - "run recomputes its phase's own code through it only to refuse a node " - "whose _code parameter differs " - "(graph_terminal.UK_FULL_GATE_PHASE_CODE); that parameter is computed " - "through it too and node_key hashes it, so an edit that changes what " - "it returns changes the parameter and the check alike, and every key " - "(graph.kernel's core reason)" - ), ("microcosm.graph.kernel", "KernelResult"): ( "the frozen record a kernel returns its outputs in, which holds what " "the kernel puts in it and computes nothing; the executor validates " @@ -1195,6 +1186,19 @@ def changed_module_bytes(path): "none or another; the stages that take an engine call it before " "computing" ), + ("microcosm.build.uk_runtime.gate_phase_codes", "require_current_phase_code"): ( + "reads its node's _code parameter and the phase's own code, " + "which it recomputes through source_hash from its battery's table, and " + "refuses the node unless they match; the spine gates' and the full " + "gates' run call it before they read anything of their node but its " + "parameters (test_each_gate_kernel_checks_its_phase_code_first). A " + "phase's own code reaches its key through that parameter, which the " + "graph construction computes through source_hash too and node_key " + "hashes, so no gate kernel's hash binds the module " + "(test_no_uk_graph_kernel_hashes_the_phase_codes), and an edit that " + "changes what source_hash returns changes the parameter and the check " + "alike (microcosm#931 ruling 3)" + ), } #: Names a stage reads from a module that neither its node nor any node #: upstream of it hashes, each with why an edit to them cannot leave the @@ -1992,7 +1996,8 @@ class the battery's kernel is built with and the artifacts that hand it class _Battery(NamedTuple): """A gate battery: the manifest its kernels evaluate, the kernel each - phase runs under, and the table of each phase's own code.""" + phase runs under, and the table of each phase's own code + (``gate_phase_codes``).""" gates: object phases: Mapping[str, str] @@ -2005,7 +2010,7 @@ def _gate_batteries() -> MappingProxyType: whose one kernel runs both spine phases of the manifest it is bound to, and the full build's preflight and calibrated gates, each running the one phase it declares of the full build's manifest.""" - from microcosm.build.uk_runtime import graph_evidence, graph_terminal + from microcosm.build.uk_runtime import gate_phase_codes from microcosm.build.uk_runtime.full_gates import uk_full_gate_manifest kernels = _uk_graph_kernels() @@ -2016,14 +2021,14 @@ def _gate_batteries() -> MappingProxyType: "spine": _Battery( spine.gates, MappingProxyType({phase: spine.ref for phase in spine.gates.phases}), - graph_evidence.UK_SPINE_GATE_PHASE_CODE, + gate_phase_codes.UK_SPINE_GATE_PHASE_CODE, ), "full": _Battery( uk_full_gate_manifest(), MappingProxyType( {phase: ref for ref in full for phase in kernels[ref].phases} ), - graph_terminal.UK_FULL_GATE_PHASE_CODE, + gate_phase_codes.UK_FULL_GATE_PHASE_CODE, ), } ) @@ -2239,15 +2244,11 @@ def test_each_gate_battery_phase_keys_on_its_own_reach() -> None: chain on, carries nothing only the transferred phase computes through, and the preflight's, which the dense solve chains on, nothing only the terminal phase does. An edit to such a module still re-keys the sibling - phase where a node upstream of it hashes the module (graph_evidence's and - graph_terminal's phase-code docstrings name where). What a phase computes - through is its walk (``_battery_phase_walk``), which the kernel rule's - test holds each phase's key to cover.""" - from microcosm.build.uk_runtime.graph_evidence import ( - UK_SPINE_GATE_PHASE_CODE, - add_uk_spine_gate_nodes, - ) - + phase where a node upstream of it hashes the module (gate_phase_codes' + docstrings name where). What a phase computes through is its walk + (``_battery_phase_walk``), which the kernel rule's test holds each phase's + key to cover. That each gate node carries its own phase's code, and only + its own, is held in test_uk_graph_evidence and test_uk_graph_terminal.""" kernels = _uk_graph_kernels() rule = _kernel_rule_walks() problems: dict[tuple[str, str], object] = {} @@ -2274,19 +2275,118 @@ def test_each_gate_battery_phase_keys_on_its_own_reach() -> None: if (phase in battery.codes) != bool(reach - shared): problems[(name, f"{phase} code")] = sorted(reach - shared) assert not problems, problems - # The spine gate nodes carry each phase's own code, and only their own; - # the full gate nodes' are held in test_uk_graph_terminal. - gated = add_uk_spine_gate_nodes( - uk_spine_graph(), spec=load_country_spec("uk"), engine_identity="test" + + +def test_each_gate_kernel_checks_its_phase_code_first() -> None: + """Each gate battery's kernel refuses a stale or missing ``_code`` + before it reads anything of its node but its parameters, as the stale-code + check's refusal-only reason says (microcosm#931 ruling 3): its run calls + gate_phase_codes.require_current_phase_code in a statement of its own, on + its node's parameters, the phase it evaluates and its own battery's table, + and no statement before that reads the context beyond its parameters (the + node's artifacts, sources and population). So the spine gates refuse a + stale code as the full gates do. A kernel that read a phase-code table + outside the check would compute through gate_phase_codes, which the + kernel rule's test would find unhashed, since the rule does not follow + the listed check.""" + import ast + + from microcosm.build.uk_runtime import gate_phase_codes + + check = (gate_phase_codes.__name__, "require_current_phase_code") + assert check in _REFUSAL_ONLY_HELPERS + kernels = _uk_graph_kernels() + for ref, battery in _GATE_BATTERY_KERNELS.items(): + ((owner, run),) = [ + (cls, statement) + for cls, statement, name in _kernel_body(type(kernels[ref])) + if name == "run" + ] + calls = [ + index + for index, item in enumerate(run.body) + if isinstance(item, ast.Expr) + and isinstance(item.value, ast.Call) + and _named(owner.__module__, item.value.func, _HELPER_SCOPE) == {check} + ] + assert len(calls) == 1, ref + (index,) = calls + assert ast.unparse(run.body[index].value) == ( + f"require_current_phase_code(context.params, phase, battery={battery!r})" + ), ref + before = run.body[:index] + assert "phase = str(context.params['phase'])" in map(ast.unparse, before), ref + for item in before: + params = { + id(sub.value) + for sub in ast.walk(item) + if isinstance(sub, ast.Attribute) + and isinstance(sub.value, ast.Name) + and sub.attr == "params" + } + assert all( + id(sub) in params + for sub in ast.walk(item) + if isinstance(sub, ast.Name) and sub.id == "context" + ), (ref, ast.unparse(item)) + # The phase the run evaluates is the one it checked. + evaluated = [ + sub + for item in run.body[index + 1 :] + for sub in ast.walk(item) + if isinstance(sub, ast.Call) + and getattr(sub.func, "id", None) == "evaluate_phase" + ] + assert evaluated and all( + "phase" in {getattr(arg, "id", None) for arg in sub.args} + or any( + keyword.arg == "phase" and getattr(keyword.value, "id", None) == "phase" + for keyword in sub.keywords + ) + for sub in evaluated + ), ref + + +def test_no_uk_graph_kernel_hashes_the_phase_codes() -> None: + """The phase codes and their stale-code check live in gate_phase_codes, + which no UK graph kernel's key covers, the gate kernels' implementation + hashes among them (microcosm#931 ruling 3a). A phase's own code reaches + its key through the ``_code`` parameter alone, so an edit to what + one phase alone computes through moves that parameter and re-keys no node + whose kernel hashes graph_evidence or graph_terminal, which define the + gate kernels, and neither of those carries the tables, the code + functions or the check.""" + from microcosm.build.uk_runtime import ( + gate_phase_codes, + graph_evidence, + graph_terminal, ) - for phase in _gate_batteries()["spine"].phases: - params = gated.node(f"spine.gates.{phase}").params - code = UK_SPINE_GATE_PHASE_CODE.get(phase) - assert {key for key in params if key.endswith("_code")} == ( - {f"{phase}_code"} if code is not None else set() - ), phase - if code is not None: - assert params[f"{phase}_code"] == code(), phase + + kernels = _uk_graph_kernels() + for ref in _GATE_BATTERY_KERNELS: + assert gate_phase_codes.__name__ not in _hashed_modules(kernels[ref]), ref + keyed = sorted( + label + for walks in _kernel_rule_walks().values() + for label, walk in walks.items() + if gate_phase_codes.__name__ in walk.key + ) + assert not keyed, keyed + batteries = _gate_batteries() + assert batteries["spine"].codes is gate_phase_codes.UK_SPINE_GATE_PHASE_CODE + assert batteries["full"].codes is gate_phase_codes.UK_FULL_GATE_PHASE_CODE + moved = { + "UK_SPINE_GATE_PHASE_CODE", + "UK_FULL_GATE_PHASE_CODE", + "require_current_phase_code", + *( + code.__name__ + for battery in batteries.values() + for code in battery.codes.values() + ), + } + for module in (graph_evidence, graph_terminal): + assert not moved & set(vars(module)), module.__name__ def test_each_gate_battery_walks_the_engine_its_build_hands_it() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py index 224cba22dc..658ce82b72 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py @@ -374,9 +374,9 @@ def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions( # the energy rake, so it alone keys on the module that sets the level, and # the assembled gate, which every later spine stage waits on, does not # (the re-review of microcosm#1420, E; microcosm#931 ruling 6). - from microcosm.build.uk_runtime import energy_pricing, graph_evidence + from microcosm.build.uk_runtime import energy_pricing, gate_phase_codes - codes = graph_evidence.UK_SPINE_GATE_PHASE_CODE + codes = gate_phase_codes.UK_SPINE_GATE_PHASE_CODE assert transferred.params["transferred_code"] == codes["transferred"]() assert assembled.params["assembled_code"] == codes["assembled"]() assert "transferred_code" not in assembled.params @@ -384,12 +384,12 @@ def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions( hashed = [] with monkeypatch.context() as patch: patch.setattr( - graph_evidence, + gate_phase_codes, "source_hash", lambda *objects: hashed.append(objects) or "", ) - graph_evidence.uk_spine_transferred_only_code_hash.__wrapped__() - graph_evidence.uk_spine_assembled_only_code_hash.__wrapped__() + gate_phase_codes.uk_spine_transferred_only_code_hash.__wrapped__() + gate_phase_codes.uk_spine_assembled_only_code_hash.__wrapped__() transferred_modules, assembled_modules = hashed assert energy_pricing in transferred_modules assert energy_pricing not in assembled_modules @@ -574,6 +574,87 @@ def test_sample_admission_holds_under_the_default_atomic_law(synthetic_smoke): ) +class _ArtifactReadError(Exception): + """A spine gate read its node's artifacts.""" + + +class _UnreadArtifacts(dict): + """Stage evidence a refused spine gate must not read: any read raises.""" + + def get(self, *args, **kwargs): + raise _ArtifactReadError(args) + + def __getitem__(self, key): + raise _ArtifactReadError(key) + + +def _spine_gate_context(phase: str, **changes): + """A spine gate kernel and a context with the node's declared parameters, + ``changes`` applied (a ``None`` value removes the parameter), and + artifacts any read of which raises :class:`_ArtifactReadError`.""" + from microcosm.build.uk_runtime.graph_evidence import ( + UKSpineGateKernel, + uk_spine_gate_manifest, + ) + + country = load_country_spec("uk") + gated = add_uk_spine_gate_nodes( + uk_spine_graph(country), spec=country, engine_identity="test-engine" + ) + params = dict(gated.node(f"spine.gates.{phase}").params) + for key, value in changes.items(): + if value is None: + params.pop(key, None) + else: + params[key] = value + kernel = UKSpineGateKernel( + gates=uk_spine_gate_manifest(country), + engine=None, + engine_identity="test-engine", + ) + return kernel, SimpleNamespace(params=params, artifacts=_UnreadArtifacts()) + + +@pytest.mark.parametrize("phase", ["assembled", "transferred"]) +@pytest.mark.parametrize("code", ["0" * 64, None]) +def test_each_spine_gate_phase_refuses_a_stale_or_missing_phase_code(phase, code): + """Each spine gate phase's node carries its own code, which the kernel + checks before it reads anything of its node but its parameters, as the + full gates check theirs (microcosm#931 ruling 3b): a graph declaration + replayed from another process cannot key a spine gate on stale phase + code. The node's own declared code passes the check, and the kernel goes + on to read its artifacts.""" + kernel, declared = _spine_gate_context(phase) + with pytest.raises(_ArtifactReadError): + kernel.run(declared) + kernel, context = _spine_gate_context(phase, **{f"{phase}_code": code}) + with pytest.raises(ValueError, match=f"UK spine gate {phase}-only code"): + kernel.run(context) + + +def test_each_spine_gate_node_carries_its_own_phase_code() -> None: + """The assembled and transferred gate nodes each carry their phase's own + code, and only their own (microcosm#931 ruling 6); the full gate nodes' + are held in test_uk_graph_terminal.""" + from microcosm.build.uk_runtime.gate_phase_codes import UK_SPINE_GATE_PHASE_CODE + from microcosm.build.uk_runtime.graph_evidence import uk_spine_gate_manifest + + country = load_country_spec("uk") + gated = add_uk_spine_gate_nodes( + uk_spine_graph(country), spec=country, engine_identity="test-engine" + ) + phases = uk_spine_gate_manifest(country).phases + assert set(phases) == {"assembled", "transferred"} + for phase in phases: + params = gated.node(f"spine.gates.{phase}").params + code = UK_SPINE_GATE_PHASE_CODE.get(phase) + assert {key for key in params if key.endswith("_code")} == ( + {f"{phase}_code"} if code is not None else set() + ), phase + if code is not None: + assert params[f"{phase}_code"] == code(), phase + + def test_the_spine_gate_kernel_keys_on_its_checks_not_the_energy_level(monkeypatch): # The re-review of microcosm#1420, E: one kernel hash serves both phases, # so it covers the modules both read and not energy_pricing, which only diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index 13a7ed967e..1733c07896 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -399,10 +399,12 @@ def test_full_preflight_persists_real_source_failures_without_matrix_diagnostics from microcosm.build.gate_battery import _gates_manifest_payload from microcosm.build.uk_runtime import full_gates + from microcosm.build.uk_runtime.gate_phase_codes import ( + uk_preflight_only_code_hash, + ) from microcosm.build.uk_runtime.graph_terminal import ( UKFullPreflightGateKernel, decode_full_gate_report, - uk_preflight_only_code_hash, ) from microcosm.graph.canonical import canonical_json @@ -970,11 +972,9 @@ def _run_final_gate_kernel(monkeypatch, *, brma_row: bool = False, solved=None): from microcosm.build.gate_battery import _gates_manifest_payload from microcosm.build.uk_runtime import full_gates, geography_ladder + from microcosm.build.uk_runtime.gate_phase_codes import uk_terminal_only_code_hash from microcosm.build.uk_runtime.graph_targets import registry_payload - from microcosm.build.uk_runtime.graph_terminal import ( - UKFullGateKernel, - uk_terminal_only_code_hash, - ) + from microcosm.build.uk_runtime.graph_terminal import UKFullGateKernel from microcosm.calibrate import ( TargetRegistry, TargetSet, @@ -1472,6 +1472,7 @@ def test_the_gate_kernel_keys_the_solve_without_the_high_value_homes_code( # surface reads, are in the kernel's hash, and the target kernels hash it # too (microcosm#931 ruling 6). from microcosm.build.uk_runtime import ( + gate_phase_codes, graph_terminal, high_value_home_gates, high_value_homes, @@ -1485,6 +1486,7 @@ def recording(*objects, **kwargs): return real(*objects, **kwargs) monkeypatch.setattr(graph_terminal, "source_hash", recording) + monkeypatch.setattr(gate_phase_codes, "source_hash", recording) # The preflight kernel inherits the calibrated kernel's code hash. for kernel in ( graph_terminal.UKFullGateKernel, @@ -1498,12 +1500,12 @@ def recording(*objects, **kwargs): # The hash is memoised per process; its undecorated body hashes the # high-value homes evidence (and the diagnostics publication code, next # test), and not what the kernel's hash holds. - graph_terminal.uk_terminal_only_code_hash.__wrapped__() + gate_phase_codes.uk_terminal_only_code_hash.__wrapped__() assert high_value_home_gates.__name__ in hashed[-1] assert high_value_homes.__name__ not in hashed[-1] - first = graph_terminal.uk_terminal_only_code_hash() + first = gate_phase_codes.uk_terminal_only_code_hash() calls = len(hashed) - assert graph_terminal.uk_terminal_only_code_hash() == first + assert gate_phase_codes.uk_terminal_only_code_hash() == first assert len(hashed) == calls @@ -1527,7 +1529,11 @@ def test_the_calibrated_node_keys_on_the_schema_9_publication_code_not_the_solve import microcosm.calibrate.hierarchy_labels as hierarchy_labels_module import microcosm.diagnostics.catalogs as catalogs_module import microcosm.diagnostics.schema as diagnostics_schema_module - from microcosm.build.uk_runtime import graph_terminal, high_value_home_gates + from microcosm.build.uk_runtime import ( + gate_phase_codes, + graph_terminal, + high_value_home_gates, + ) hashed_module_names: list[set[str]] = [] real_source_hash = graph_terminal.source_hash @@ -1537,6 +1543,7 @@ def record_hashed_modules(*objects, **dependencies): return real_source_hash(*objects, **dependencies) monkeypatch.setattr(graph_terminal, "source_hash", record_hashed_modules) + monkeypatch.setattr(gate_phase_codes, "source_hash", record_hashed_modules) publication = { catalogs_module.__name__, diagnostics_schema_module.__name__, @@ -1551,7 +1558,7 @@ def record_hashed_modules(*objects, **dependencies): ).implementation_hash() assert not hashed_module_names[-1] & publication, kernel.__name__ assert geography_constants_module.__name__ in hashed_module_names[-1] - graph_terminal.uk_terminal_only_code_hash.__wrapped__() + gate_phase_codes.uk_terminal_only_code_hash.__wrapped__() assert {high_value_home_gates.__name__, *publication} <= hashed_module_names[-1] assert geography_constants_module.__name__ not in hashed_module_names[-1] @@ -1560,7 +1567,9 @@ def record_hashed_modules(*objects, **dependencies): @pytest.mark.parametrize("code", ["0" * 64, None]) def test_each_gate_phase_refuses_a_stale_or_missing_phase_code(phase, code) -> None: """Each full gate phase's node carries its own code, which the kernel - checks before it evaluates anything (microcosm#931 ruling 6).""" + checks before it evaluates anything (microcosm#931 ruling 6), through the + check the spine gates share (gate_phase_codes.require_current_phase_code; + ruling 3).""" from types import SimpleNamespace from microcosm.build.uk_runtime.graph_terminal import ( @@ -1575,7 +1584,7 @@ def test_each_gate_phase_refuses_a_stale_or_missing_phase_code(phase, code) -> N if code is not None: params[f"{phase}_code"] = code context = SimpleNamespace(params=params, artifacts={}) - with pytest.raises(ValueError, match=f"{phase}-only code"): + with pytest.raises(ValueError, match=f"UK full gate {phase}-only code"): kernel.run(context) @@ -1606,7 +1615,7 @@ def test_the_dense_gates_refuse_references_of_two_builds() -> None: def test_each_full_gate_node_carries_its_own_phase_code() -> None: """The preflight and calibrated gate nodes each carry their phase's own code, and only their own (microcosm#931 ruling 6).""" - from microcosm.build.uk_runtime.graph_terminal import UK_FULL_GATE_PHASE_CODE + from microcosm.build.uk_runtime.gate_phase_codes import UK_FULL_GATE_PHASE_CODE graph = _gate_nodes() for node, phase in ( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index c49144f0d0..1dbde579b9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -315,14 +315,22 @@ def test_an_edit_to_calibration_run_leaves_the_preflight_and_the_solve( ): """The full gates' preflight computes through the spine and local gate scopes and the scoped gate manifest, which moved unchanged from - calibration_run into gate_scopes, so its key, and with it the dense - solve's, hashes the small module and not the runner (microcosm#931), as - the target kernels hash ledger_surface. Neither full gate kernel's own - hash nor the preflight's own code moves on an edit to the runner. The - calibrated gates' own code binds it, for the admin totals and the CGT - projection artifact their gate context is built with, so of the two - phases only the calibrated gates re-key.""" - from microcosm.build.uk_runtime import calibration_run, gate_scopes, graph_terminal + calibration_run into gate_scopes, so the full gate kernels hash the small + module and not the runner (microcosm#931), as the target kernels hash + ledger_surface. Neither full gate kernel's own hash nor the preflight's + own code moves on an edit to the runner. The calibrated gates' own code + binds it, for the admin totals and the CGT projection artifact their gate + context is built with. So in the in-graph build an edit to the runner + re-keys the calibrated gates and leaves the preflight and the solve keyed + on it. On a bound spine the checkpoint, whose kernel hashes the runner, + still re-keys the preflight and the solve by key chaining, which this + test leaves aside.""" + from microcosm.build.uk_runtime import ( + calibration_run, + gate_phase_codes, + gate_scopes, + graph_terminal, + ) kernels = [ kind(coverage_engine=object(), engine_identity="test") @@ -331,8 +339,8 @@ def test_an_edit_to_calibration_run_leaves_the_preflight_and_the_solve( graph_terminal.UKFullGateKernel, ) ] - preflight_code = graph_terminal.uk_preflight_only_code_hash.__wrapped__ - terminal_code = graph_terminal.uk_terminal_only_code_hash.__wrapped__ + preflight_code = gate_phase_codes.uk_preflight_only_code_hash.__wrapped__ + terminal_code = gate_phase_codes.uk_terminal_only_code_hash.__wrapped__ for kernel in kernels: hashed = hashed_module_names(kernel, graph_terminal) assert gate_scopes.__name__ in hashed, kernel.ref From fce2f8613dfc672d16fd1761e45f5afdc16d8f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:12:45 +0000 Subject: [PATCH 050/108] Take frs_spine and frs_relationships off the target kernels' keys (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María ruled on 9 Nov (late) that the annuity and trust column's name moves into a small module of its own (4a) and that ledger_targets' household-type check is listed refusal-only (4b), so that frs_spine and frs_relationships leave the target kernels' hashes. 08b437a8e bound both only because of the rule's limits (run A findings CM8 and TGH7), and on a bound spine each cost a national re-solve. This commit does both. It moves keys only. - The column. uk_runtime/annuity_trust_column.py holds ANNUITY_AND_TRUST_INCOME_COLUMN, unchanged, and nothing else. frs_spine, measure_simulation, spi_spine and spi_income import it from there, and frs_spine no longer defines it (its __all__ never listed it). The resolver reads zeros where a frame lacks the column, so the name cannot be listed as unbound: the measures and the national problem hash the small module in place of frs_spine. The frs_spine root, which writes the column and has no node upstream, binds it, as the stage rule asks, and so does hmrc_spi_income_spine, which zeros it on the rows it draws and could chain on the root's key. - The check. test_uk_graph lists ledger_targets._assert_household_type_bindings_declared in _REFUSAL_ONLY_HELPERS, the eighth entry. It compares the contract's ons_household_type bindings with the ONS household type domain and refuses a binding outside it or composition rows that do not cover it once each. It writes only into the dict and set it creates, which the shared writes_nothing counts as local since 8c10ecc22 and 8f6c36917. Its one caller, compile_uk_target_registry, calls it before the loop that compiles the references, and the compilation and the national targets reach it there, through full_targets' two loaders. The compilation's own hash keeps chronicle_epoch alone. The national targets compute through nothing their shared hash leaves out, so their own implementation hash goes. The shared rule module is unchanged. - Tests. test_the_household_type_check_refuses_before_the_targets_compile holds the entry's reason: the check's one caller, its statement before the compile loop, both kernels reaching the caller with no walk meeting frs_relationships or key hashing it, and the kernels that hash it for their own reasons keeping it (the frs_relationships stage and the assembled spine gate's own code). test_every_reader_of_the_annuity_and_trust_column_hashes_its_module checks that the small module holds the name alone, that no other uk_runtime module defines it or imports it from elsewhere, that every stage and kernel whose walk reads it hashes the module, and that the measures and the national problem meet nothing in frs_spine and leave it out of their keys. test_uk_target_compile_modules checks that an edit to frs_spine.py or frs_relationships.py leaves each of the six target and problem kernels' hashes alone, and that an edit to the small module moves the measures' and the national problem's. - Texts. The binding comments in graph_targets, graph_national and graph_kernels, and frs_spine's and measure_simulation's comments on the column. The helper-bindings changelog fragment's target bullets, its refusal-only count and its What moves. test_uk_target_compile_modules and docs/uk-full-build-graph.md stated neither binding, so neither needed a correction. What moves: node keys only. No value, declaration or generated file moves, and neither the solve nor the country fingerprint does. - 68 of the 99 UK kernel hashes move, exactly those that hash a file this commit edits: the 57 spine kernels, which all hash graph_kernels.py (20 of them frs_spine.py too); the six target and problem kernels; the full preflight and calibrated gates and the rotated holdout, which hash graph_targets.py; and the national gates and readback, which hash graph_national.py. Six kernels' hashed modules change: the create kernel and hmrc_spi_income_spine gain annuity_trust_column, the measures and the national problem swap frs_spine for it, and the compilation and the national targets lose frs_relationships, which now only the frs_relationships stage's kernel and the assembled gate's own code hash. The other 31 keep theirs, the bound spine, the dense population, calibration and size kernels, the export kernels and the four atomic geography kernels among them. No transport kernel moves. - Every node of the hermetic and gated spines (79 and 81) and of the in-graph build (113) re-keys, since the root's module changed. On a bound spine the local line re-keys 26 of its 33 nodes (the checkpoint, the sample, normalize, expand and the three geography support nodes keep theirs) and the national line 5 of 6 (all but the checkpoint). - After this, by simulated edits (a comment line through each module's __file__), counted as in-graph build / local line on a bound spine / national line: frs_spine.py re-keys 113, 0 and 0 nodes (113, 12 and 4 at the parent) and frs_relationships.py 108, 0 and 0 (108, 14 and 5), so on a bound spine neither line re-solves for either; annuity_trust_column.py re-keys 113, 12 and 4. - uk_spine.json and the H2 fixture hold stage contract hashes only, so neither regenerates. source_stages.json and the coverage manifest are current (both --check pass). Tests (engine-free; code facts and synthetic values): test_uk_graph and test_uk_target_compile_modules pass (119 tests), and with them 20 related files, the shared rule and transport rule tests among them, pass (838 tests). Through the engine slot, engine/uk/test_uk_full_target_graph passes (8 tests). Eleven mutations on a scratch copy are caught, each failing exactly the tests expected: - the column defined in frs_spine again and read from there by the measures (the kernel rule finds frs_spine's name outside both keys); - the measures importing it through frs_spine, which only re-exports it (the rule follows it to the small module; the reader test refuses it); - the root, the SPI income stage, the measures or the national problem no longer hashing the small module; - the household-type check unlisted while frs_relationships stays unbound (the kernel rule finds frs_relationships unhashed in the compilation's and the national targets' walks); - the check writing into what it is handed, or running after the compile loop; - the compilation hashing frs_relationships again; - the national targets reading the domain themselves. Ruff is clean on the 10 edited Python files. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-helper-bindings.changed.md | 11 +- .../build/uk_runtime/annuity_trust_column.py | 24 +++ .../microcosm/build/uk_runtime/frs_spine.py | 11 +- .../build/uk_runtime/graph_kernels.py | 10 +- .../build/uk_runtime/graph_national.py | 48 +++--- .../build/uk_runtime/graph_targets.py | 29 ++-- .../build/uk_runtime/measure_simulation.py | 12 +- .../microcosm/build/uk_runtime/spi_income.py | 4 +- .../microcosm/build/uk_runtime/spi_spine.py | 4 +- .../tests/engine_free/uk/test_uk_graph.py | 146 ++++++++++++++++++ .../uk/test_uk_target_compile_modules.py | 45 ++++++ 11 files changed, 289 insertions(+), 55 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 341340eda4..663efad862 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -1,7 +1,7 @@ Every UK spine stage now hashes each first-party module its transform computes through, and each stage that calls the engine also each one the injected engine computes through for it. Every other UK graph kernel hashes each one its own code computes through, the target and problem kernels among them, and each phase of a gate battery each one its own gates compute through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. - **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage, and for a stage that calls the engine also at the engine the builds inject (see The engine), and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports, module attributes and `type` aliases, which read as assignments of their values. So do the names `microcosm.graph` exports lazily through its module `__getattr__`: the walk resolves each to the runtime module that defines it, from the table that `__getattr__` reads, and a test checks that every first-party module with a module `__getattr__` is listed. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: - - seven listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels), and one is the gate kernels' stale phase-code check (see Gate batteries); + - eight listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels), one is the gate kernels' stale phase-code check (see Gate batteries), and one is the target compilation's household-type check (see Target and problem kernels); - the stage's own declaration (`country_spec`, `source_manifest` and the spec engine), whose content the node key already hashes as the stage contract; - the listed core modules every node runs through (see Scope); - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; two such names are listed. @@ -66,9 +66,10 @@ Every UK spine stage now hashes each first-party module its transform computes t - The national gates evaluate whichever gates their node's scope names, so their hash binds everything the gate registry's bindings compute through. That is 50 modules, which `battery_bindings.UK_GATE_REGISTRY_MODULES` lists and a test holds exactly to the registry's derived reach. Their stop at the registry, the one held kernel stop, goes. - **Target and problem kernels.** As María ruled on 9 Nov, the derived rule replaces microcosm#1497's per-kernel import record for the full build's target compilation, selection, measures and problem and the national targets and problem, which that record held until now. No kernel stays held. - `test_uk_target_compile_modules` loses the record (`UNCLASSIFIED_IMPORTS`), its kernel groups and the check that read them. It keeps the shared list's closure check, the tests of what the kernels hash and its hash inputs, the seven above among them. A new check covers what the derived rule does not: each hash input is still imported where listed, and a kernel hashes one only where another module it hashes imports it too. So no kernel hashes the compile list or the solve fingerprint's module, and an edit to either re-keys none. - - What all four full kernels compute through goes in the hash they share, and what only one computes through goes in its own implementation hash, as for the size search and refit. The shared hash gains the canonical encoder. The compilation adds `chronicle_epoch`, whose schema versions and fact-key epochs `ledger_artifact` checks the facts against and records, and `frs_relationships`, whose ONS household type domain `ledger_targets` checks the household-type bindings against. The measures add the 15 modules that resolving the measures and compiling their national problem compute through: `graph_population` and `national_frame`, for the frame they rebuild; `atomic_area_support`, `frame.materialize` and the engine adapter, for the engine's scratch frame; `dfe_spring_term`, `frs_annual_income`, `hmrc_income`, `income_tax_rate_classes`, `lcfs_consumption` and `uc_target_measurements`, for the resolver's inputs and measures; `chronicle_epoch`, for the vendored facts' schema checks; `national_calibration`, for the measure inputs it injects; `local_rowwise`, for the national rows; and `calibrate.monetary_binding`, for the constraint matrix's check. The problem adds `graph_population`, `national_frame`, `local_geography` (the local surface matrix's area columns) and `calibrate.monetary_binding`. - - The national targets' and problem's shared hash gains `chronicle_epoch`, `graph_targets` (the register codec) and the canonical encoder. The national targets add `frs_relationships`. The national problem adds the measures' modules but `chronicle_epoch` and `national_calibration`, which its shared hash already binds, and adds `diagnostics` and `gate_battery`, whose target geography levels and JSON-safe values its payload records. - - The measures and the national problem also bind `frs_spine` for one name, the annuity and trust column. The resolver reads that column where the frame carries it and reads zeros where it does not, so on the national line's bound spine a renamed constant would change the problem without re-keying it. The clone index columns' name stays listed for them: the dense measures read it only to split their frame into the engine blocks they are asked for, and refuse a frame without it, and their population is the expand's; the national problem resolves its measures in one block, which never reads it. A test checks these facts. + - What all four full kernels compute through goes in the hash they share, and what only one computes through goes in its own implementation hash, as for the size search and refit. The shared hash gains the canonical encoder. The compilation adds `chronicle_epoch`, whose schema versions and fact-key epochs `ledger_artifact` checks the facts against and records. The measures add the 15 modules that resolving the measures and compiling their national problem compute through: `graph_population` and `national_frame`, for the frame they rebuild; `atomic_area_support`, `frame.materialize` and the engine adapter, for the engine's scratch frame; `dfe_spring_term`, `frs_annual_income`, `hmrc_income`, `income_tax_rate_classes`, `lcfs_consumption` and `uc_target_measurements`, for the resolver's inputs and measures; `chronicle_epoch`, for the vendored facts' schema checks; `national_calibration`, for the measure inputs it injects; `local_rowwise`, for the national rows; and `calibrate.monetary_binding`, for the constraint matrix's check. The problem adds `graph_population`, `national_frame`, `local_geography` (the local surface matrix's area columns) and `calibrate.monetary_binding`. + - The national targets' and problem's shared hash gains `chronicle_epoch`, `graph_targets` (the register codec) and the canonical encoder. The national targets compute through nothing else, so they have no hash of their own. The national problem adds the measures' modules but `chronicle_epoch` and `national_calibration`, which its shared hash already binds, and adds `diagnostics` and `gate_battery`, whose target geography levels and JSON-safe values its payload records. + - The measures and the national problem also bind the annuity and trust column's name. The resolver reads that column where the frame carries it and reads zeros where it does not, so on the national line's bound spine a renamed constant would change the problem without re-keying it, and the name cannot be listed as unbound. As María ruled on 9 Nov (late), the name moved unchanged from `frs_spine` into a module of its own, `uk_runtime.annuity_trust_column`, which holds nothing else, and `frs_spine`, `measure_simulation`, `spi_spine` and `spi_income` each import it from there. So the two kernels bind that module, not `frs_spine`, whose every edit would otherwise re-key them. The `frs_spine` root, which writes the column, binds it too, and so does `hmrc_spi_income_spine`, which zeros it on the rows it draws, though that stage could chain on the root's key; a test holds every stage and kernel whose code reads the name to its module. The clone index columns' name stays listed for them: the dense measures read it only to split their frame into the engine blocks they are asked for, and refuse a frame without it, and their population is the expand's; the national problem resolves its measures in one block, which never reads it. A test checks these facts. + - `ledger_targets`' household-type check is listed refusal-only, as María ruled on 9 Nov (late). It compares the contract's `ons_household_type` bindings with the ONS household type domain and refuses a binding outside it or composition rows that do not cover it once each. It writes only into the dict and set it creates, which the shared refusal check counts as local. Its one caller, `compile_uk_target_registry`, calls it before the loop that compiles the references, on the compilation's and the national targets' paths alike. So neither kernel binds `frs_relationships`, which the rule asked of them only through this check. A test checks the check's place in its caller, that neither kernel computes through `frs_relationships` another way, and that the kernels that hash it for their own reasons keep it: the `frs_relationships` stage, whose transform it defines, and the assembled spine gate, whose own code holds the domain its enum gate checks. - No kernel drops a module it hashed before, though each still hashes some modules its walk does not reach. The full kernels' shared hash still binds the whole compile list and the modules beside it in all four, so the selection's key covers 51 modules while its code computes through 6. - `transport/compile_modules.py`'s reason for leaving the graph runtime off the transport compile list said that no kernel in the repository hashes a `microcosm.graph` module, which was untrue. It now says what the listed modules use those modules for, that the compile kernels hash none of them, and that other kernels hash the canonical encoder, each UK graph kernel whose code computes through it among them. No kernel hashes that file, so the edit re-keys nothing. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. The target and problem kernels' keys move once with their new bindings. `graph_targets.py` and `graph_national.py`, which define them, changed with them, and the preflight and calibrated gates, the rotated holdout and the national gates and readback each hash one or both of those files too. So in the full build the target compilation, selection, measures and problem, the pool checkpoint (`uk.full.pool`), the preflight, the rotated holdout, and the dense solve and every node after it re-key, and on the national line every node after its bound spine root; no spine node and no transport node re-keys with them. The phase codes' move re-keys once more every node whose kernel hashes `graph_evidence.py`, `graph_terminal.py`, `gate_scopes.py` or `calibration_run.py`, which is every node of every UK graph, the bound spine of the local and national lines included, whose kernel hashes the last two; of the phase codes only `terminal_code` moves, since it hashes `calibration_run`. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. The target and problem kernels' keys move once with their new bindings. `graph_targets.py` and `graph_national.py`, which define them, changed with them, and the preflight and calibrated gates, the rotated holdout and the national gates and readback each hash one or both of those files too. So in the full build the target compilation, selection, measures and problem, the pool checkpoint (`uk.full.pool`), the preflight, the rotated holdout, and the dense solve and every node after it re-key, and on the national line every node after its bound spine root; no spine node and no transport node re-keys with them. The phase codes' move re-keys once more every node whose kernel hashes `graph_evidence.py`, `graph_terminal.py`, `gate_scopes.py` or `calibration_run.py`, which is every node of every UK graph, the bound spine of the local and national lines included, whose kernel hashes the last two; of the phase codes only `terminal_code` moves, since it hashes `calibration_run`. The column's move and the listed household-type check re-key once more every node whose kernel hashes a file they edit (`frs_spine.py`, `graph_kernels.py`, `graph_targets.py`, `graph_national.py`, `measure_simulation.py`, `spi_spine.py` or `spi_income.py`): every spine node and every node of the in-graph build, 26 of the local line's 33 nodes on a bound spine, and the national line's every node after its bound spine root. After them, on a bound spine an edit to `frs_spine.py` or `frs_relationships.py` re-keys no node of the local or the national line, where it re-keyed 12 and 4 of their nodes for `frs_spine.py` and 14 and 5 for `frs_relationships.py`, so neither line re-solves for one; in the in-graph build either still re-keys its stage and every node after it, as before. An edit to `annuity_trust_column.py` re-keys the spine root, the measures and the national problem, and what follows them. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py new file mode 100644 index 0000000000..06f3b2a613 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py @@ -0,0 +1,24 @@ +"""The name of the UK spine's annuity and trust income descriptor. + +FRS rows keep the payments of an annuity not bought with pension funds and of +a trust or covenant in private_pension_income, where policyengine-uk's means +tests, household income and tax read them. SPI Table 3.7 counts both as other +income (note 6), so the spine stores the part they make of +private_pension_income apart, under this name: a calibration-only descriptor +that the Table 3.7 other-income measure adds (microcosm#1428 review, finding +1). frs_spine writes it, the SPI income stage zeros it on the rows it draws, +and measure_simulation's resolver reads it. + +The resolver reads zeros where a frame lacks the column, so a renamed constant +would change the measures without re-keying them, and every node whose code +reads the name binds this module. It holds the name alone, nothing that +computes, so the target and problem kernels key on the name without keying on +frs_spine, every edit to which would otherwise re-solve the national line on a +bound spine (microcosm#931 ruling 4). +""" + +from __future__ import annotations + +#: The descriptor: each person's annual annuity and trust payments, already +#: inside private_pension_income. No engine input reads it. +ANNUITY_AND_TRUST_INCOME_COLUMN = "annuity_and_trust_income" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index a0bcf9cb12..334c6de566 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -14,6 +14,9 @@ from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.trace import sha256_file from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime.annuity_trust_column import ( + ANNUITY_AND_TRUST_INCOME_COLUMN, +) from microcosm.build.uk_runtime.birth_month import ( UK_BIRTH_MONTH_COLUMN, uk_months_since_last_birthday, @@ -437,11 +440,11 @@ def benefits_by_recipient( #: (ITTOIA 2005 s. 717). SPI Table 3.7 counts both as other income (note 6), #: so the part they make of private_pension_income is stored apart, in #: ANNUITY_AND_TRUST_INCOME_COLUMN, a calibration-only descriptor that the -#: Table 3.7 other-income measure adds (microcosm#1428 review, finding 1). +#: Table 3.7 other-income measure adds (microcosm#1428 review, finding 1). The +#: descriptor's name lives in annuity_trust_column, which holds nothing else, +#: so the nodes that read it key on it without keying on this module +#: (microcosm#931 ruling 4). FRS_ANNUITY_AND_TRUST_PENSION_TYPES = (4, 5) -#: The descriptor: each person's annual annuity and trust payments, already -#: inside private_pension_income. No engine input reads it. -ANNUITY_AND_TRUST_INCOME_COLUMN = "annuity_and_trust_income" #: Each person's company car from the FRS job tab, the four carriers the #: company_cars stage reads to set policyengine-uk's company car inputs #: (PolicyEngine/uk-autumn-budget-dashboard-2026#316); see diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index ba26a9efb6..d70e952635 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -71,6 +71,7 @@ ) from . import ( advani_summers, + annuity_trust_column, birth_month, bus_fare_pricing, bus_support_pricing, @@ -336,10 +337,13 @@ # kinds, gate outcomes and the HMRC replay report (microcosm#931). It # binds the copy-id table as the leaves do. The imputation refreshes the # annual income descriptors on frs_annual_income's share columns, and the - # evidence shares iterate HMRC's distributional inputs. + # evidence shares iterate HMRC's distributional inputs. It zeros a drawn + # row's annuity and trust descriptor, whose name it reads from + # annuity_trust_column, as the root does (microcosm#931 ruling 4). "hmrc_spi_income_spine": ( spi_income, spi_rank_transport, + annuity_trust_column, copy_id_table, employer_pension_contributions, *_FEED_FACT_MODULES, @@ -370,9 +374,11 @@ # declared radix from the table too. The root reads its tabs through # trace's digest, ages people by birth month, caps pension contributions # with their receipt, and codes council tax bands in the shared band order - # (microcosm#931). + # (microcosm#931). It writes the annuity and trust descriptor under the + # name annuity_trust_column holds (microcosm#931 ruling 4). "frs_spine": ( uc_relationships, + annuity_trust_column, copy_id_table, birth_month, council_tax_constants, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 8e6570d956..6573d09929 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -513,9 +513,10 @@ def implementation_hash(self) -> str: # Chronicle schema versions and fact-key epochs the facts are # checked against (chronicle_epoch), the register codec # (graph_targets) and the canonical encoding of their artifacts. - # What only one of them computes through, it hashes in its own - # implementation hash below; test_uk_graph derives each kernel's - # set from what its code names. + # What only the problem computes through, it hashes in its own + # implementation hash below, and the targets compute through + # nothing else; test_uk_graph derives each kernel's set from what + # its code names. chronicle_epoch, graph_targets, canonical_json, @@ -531,23 +532,14 @@ def implementation_hash(self) -> str: class UKNationalTargetKernel(_NationalKernel): + # The national register is compiled through ledger_targets, which checks + # the household-type bindings against the ONS household type domain + # (frs_relationships) before it compiles. That check only refuses, so + # test_uk_graph lists it refusal-only, and this kernel computes through + # nothing its shared hash leaves out: it needs no hash of its own + # (microcosm#931 ruling 4). ref = "uk.full.national_targets@1" - def implementation_hash(self) -> str: - from . import frs_relationships - - # The national register is compiled through ledger_targets, which - # checks the household-type bindings against the ONS household type - # domain (frs_relationships) (microcosm#931). - return hashlib.sha256( - canonical_json( - { - "national": super().implementation_hash(), - "targets": source_hash(frs_relationships), - } - ) - ).hexdigest() - def run(self, context: KernelContext) -> KernelResult: inputs = load_uk_national_target_inputs( context.sources["uk_ledger_facts"], @@ -705,11 +697,11 @@ def implementation_hash(self) -> str: from .. import gate_battery from . import ( + annuity_trust_column, atomic_area_support, dfe_spring_term, diagnostics, frs_annual_income, - frs_spine, graph_population, hmrc_income, income_tax_rate_classes, @@ -725,22 +717,26 @@ def implementation_hash(self) -> str: # (graph_population, national_frame), the engine's scratch frame # (atomic_area_support, frame_materialize and the engine adapter's # contracts), the resolver's inputs and measures (dfe_spring_term, - # frs_annual_income, frs_spine's annuity and trust column, - # hmrc_income, income_tax_rate_classes, lcfs_consumption and - # uc_target_measurements), the national rows (local_rowwise), the - # constraint matrix's monetary check (monetary_binding), and the - # target geography levels (diagnostics) and JSON-safe values - # (gate_battery) its payload records. + # frs_annual_income, hmrc_income, income_tax_rate_classes, + # lcfs_consumption and uc_target_measurements), the national rows + # (local_rowwise), the constraint matrix's monetary check + # (monetary_binding), and the target geography levels (diagnostics) + # and JSON-safe values (gate_battery) its payload records. As the + # dense measures do, it binds the annuity and trust column's name, + # since the resolver reads zeros where a frame lacks the column. The + # name lives in annuity_trust_column, which holds nothing else, so + # this hash leaves frs_spine out, and an edit there does not re-solve + # the national line (microcosm#931 ruling 4). return hashlib.sha256( canonical_json( { "national": super().implementation_hash(), "problem": source_hash( + annuity_trust_column, atomic_area_support, dfe_spring_term, diagnostics, frs_annual_income, - frs_spine, graph_population, hmrc_income, income_tax_rate_classes, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index 3473d7c3e8..a3a88414e4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -284,18 +284,19 @@ class UKFullTargetCompilationKernel(_TargetKernel): def implementation_hash(self) -> str: from microcosm.build import chronicle_epoch - from . import frs_relationships - # The compilation also computes through the Chronicle schema versions # and fact-key epochs ledger_artifact checks the facts against and - # records in their provenance (chronicle_epoch), and the ONS household - # type domain ledger_targets checks the household-type bindings - # against (frs_relationships) (microcosm#931). + # records in their provenance (chronicle_epoch) (microcosm#931). It + # does not compute through the ONS household type domain + # (frs_relationships): ledger_targets checks the household-type + # bindings against it before it compiles, and that check only + # refuses, so test_uk_graph lists it refusal-only (microcosm#931 + # ruling 4). return hashlib.sha256( canonical_json( { "targets": super().implementation_hash(), - "compilation": source_hash(chronicle_epoch, frs_relationships), + "compilation": source_hash(chronicle_epoch), } ) ).hexdigest() @@ -502,10 +503,10 @@ def implementation_hash(self) -> str: from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from . import ( + annuity_trust_column, atomic_area_support, dfe_spring_term, frs_annual_income, - frs_spine, graph_population, hmrc_income, income_tax_rate_classes, @@ -520,22 +521,28 @@ def implementation_hash(self) -> str: # national_frame's schema (graph_population, national_frame), the # engine's scratch frame (atomic_area_support, frame_materialize and # the engine adapter's contracts), the resolver's inputs and measures - # (dfe_spring_term, frs_annual_income, frs_spine's annuity and trust - # column, hmrc_income, income_tax_rate_classes, lcfs_consumption and + # (dfe_spring_term, frs_annual_income, hmrc_income, + # income_tax_rate_classes, lcfs_consumption and # uc_target_measurements), the vendored facts' schema checks # (chronicle_epoch), the measure inputs the calibration frame adapter # injects (national_calibration), the national rows (local_rowwise) - # and the constraint matrix's monetary check (monetary_binding). + # and the constraint matrix's monetary check (monetary_binding). The + # resolver reads the annuity and trust column where the frame carries + # it and zeros where it does not, so the measures bind its name too. + # It lives in annuity_trust_column, which holds nothing else, not in + # frs_spine, which writes the column, so this hash leaves frs_spine + # out, and on a bound spine an edit there leaves the measures' key + # alone (microcosm#931 ruling 4). return hashlib.sha256( canonical_json( { "targets": super().implementation_hash(), "measures": source_hash( + annuity_trust_column, atomic_area_support, chronicle_epoch, dfe_spring_term, frs_annual_income, - frs_spine, graph_population, hmrc_income, income_tax_rate_classes, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py index 8cdbe196d7..f6cc2b68b8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -19,6 +19,9 @@ evaluate_named_term, named_term_leaves, ) +from microcosm.build.uk_runtime.annuity_trust_column import ( + ANNUITY_AND_TRUST_INCOME_COLUMN, +) from microcosm.build.uk_runtime.atomic_area_support import ( without_uk_native_alias_columns, ) @@ -42,7 +45,6 @@ SELF_EMPLOYMENT_INCOME_ANNUAL_SHARE, SELF_EMPLOYMENT_INCOME_OTHER_SPELLS, ) -from microcosm.build.uk_runtime.frs_spine import ANNUITY_AND_TRUST_INCOME_COLUMN from microcosm.build.uk_runtime.high_value_homes import ( high_value_home_mask, uk_high_value_home_thresholds, @@ -306,10 +308,10 @@ def savings_interest_outside_isas(frame: Any) -> np.ndarray: #: private_pension_income, where the engine's means tests, household income #: and tax read them (microcosm#1428). The frame stores each person's annuity #: and trust payments at the dataset's year -#: (frs_spine.ANNUITY_AND_TRUST_INCOME_COLUMN), and their share of the stored -#: private pension carries them to the measurement year on the engine's own -#: private pension path. A frame without the column (the enhanced FRS -#: reference) reads the engine's other income alone. A reserved name. +#: (annuity_trust_column.ANNUITY_AND_TRUST_INCOME_COLUMN), and their share of +#: the stored private pension carries them to the measurement year on the +#: engine's own private pension path. A frame without the column (the enhanced +#: FRS reference) reads the engine's other income alone. A reserved name. UK_SPI_OTHER_INVESTMENT_INCOME = "other_investment_income_spi_concept" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py index c4dd0fb27c..011bf3eb56 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py @@ -19,6 +19,9 @@ integer_record_keys, stable_identity_uniforms, ) +from microcosm.build.uk_runtime.annuity_trust_column import ( + ANNUITY_AND_TRUST_INCOME_COLUMN, +) from microcosm.build.uk_runtime.frs_annual_income import ( ANNUAL_SHARE_COLUMNS, EMPLOYMENT_INCOME_ANNUAL_SHARE, @@ -38,7 +41,6 @@ FRS_HMRC_UBISJA_COLUMN, ) from microcosm.build.uk_runtime.frs_spine import ( - ANNUITY_AND_TRUST_INCOME_COLUMN, FRS_LANDLORD_CARRIER_COLUMN, ISA_INTEREST_COLUMN, WEEKS_IN_YEAR, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index e33b882a6f..d1da7420f9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -12,6 +12,9 @@ from microcosm.build.gates import FitWeightRecord from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.annuity_trust_column import ( + ANNUITY_AND_TRUST_INCOME_COLUMN, +) from microcosm.build.uk_runtime.employer_pension_contributions import ( draw_employer_pension_contributions, employer_scheme_and_sector, @@ -29,7 +32,6 @@ _raw_source_person_ids, _read_raw_frs_table, ) -from microcosm.build.uk_runtime.frs_spine import ANNUITY_AND_TRUST_INCOME_COLUMN from microcosm.build.uk_runtime.frs_take_up import uk_take_up_population_policy from microcosm.build.uk_runtime.hmrc_income import ( HMRCIncomeTargetSet, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 64054544c9..0f309e527f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -1199,6 +1199,21 @@ def changed_module_bytes(path): "changes what source_hash returns changes the parameter and the check " "alike (microcosm#931 ruling 3)" ), + ( + "microcosm.build.uk_runtime.ledger_targets", + "_assert_household_type_bindings_declared", + ): ( + "compares the target contract's ons_household_type bindings with the " + "ONS household type domain (frs_relationships) and refuses a binding " + "outside it, two composition rows binding one value, or composition " + "rows that leave a value unbound, writing only into the dict and set " + "it creates. Its one caller, compile_uk_target_registry, calls it in a " + "statement of its own before the loop that compiles the references, " + "and the full build's target compilation and the national targets " + "both reach it there, so neither computes through frs_relationships " + "(test_the_household_type_check_refuses_before_the_targets_compile; " + "microcosm#931 ruling 4)" + ), } #: Names a stage reads from a module that neither its node nor any node #: upstream of it hashes, each with why an edit to them cannot leave the @@ -2389,6 +2404,137 @@ def test_no_uk_graph_kernel_hashes_the_phase_codes() -> None: assert not moved & set(vars(module)), module.__name__ +def test_the_household_type_check_refuses_before_the_targets_compile() -> None: + """The household-type check's refusal-only reason (microcosm#931 ruling + 4): ledger_targets' check is listed, its one caller is + compile_uk_target_registry, which calls it in a statement of its own, on + the contract's targets, before the loop that compiles the references, and + the full build's target compilation and the national targets both reach + that caller. Neither kernel computes through frs_relationships another + way, so neither key hashes it. The kernels that compute through it for + their own reasons still hash it: the frs_relationships stage, whose + transform it defines, and the assembled spine gate, whose own code holds + the domain its enum gate checks. Unlisted, the check would put the domain + in both kernels' walks, which the kernel rule's test would find + unhashed.""" + import ast + + from microcosm.build.uk_runtime import frs_relationships, ledger_targets + + check = (ledger_targets.__name__, "_assert_household_type_bindings_declared") + caller = (ledger_targets.__name__, "compile_uk_target_registry") + assert check in _REFUSAL_ONLY_HELPERS + build = Path(ledger_targets.__file__).parents[1] + naming = sorted( + path.relative_to(build).as_posix() + for path in build.rglob("*.py") + if check[1] in path.read_text(encoding="utf-8") + ) + assert naming == ["uk_runtime/ledger_targets.py"], naming + + def calls_check(node) -> bool: + return any( + isinstance(sub, ast.Call) + and _named(check[0], sub.func, _HELPER_SCOPE) == {check} + for sub in ast.walk(node) + ) + + definitions = _module_definitions(check[0], _HELPER_SCOPE).definitions + callers = sorted( + name + for name, node in definitions.items() + if name != check[1] and calls_check(node) + ) + assert callers == [caller[1]], callers + body = definitions[caller[1]].body + (index,) = [ + position + for position, statement in enumerate(body) + if isinstance(statement, ast.Expr) and calls_check(statement) + ] + assert ast.unparse(body[index].value) == ( + "_assert_household_type_bindings_declared(_uk_contract_targets())" + ) + assert not any(calls_check(statement) for statement in body[index + 1 :]) + loops = [ + position + for position, statement in enumerate(body) + if isinstance(statement, ast.For) + and ast.unparse(statement.iter) == "spec.target_references" + ] + assert loops and index < min(loops), (index, loops) + domain = frs_relationships.__name__ + rule = _kernel_rule_walks() + for ref in ("uk.full.target_compilation@1", "uk.full.national_targets@1"): + ((walk, key),) = rule[ref].values() + assert caller[1] in walk.reach.get(caller[0], ()), ref + assert domain not in walk.reach, ref + assert not [item for item in walk.stops if item[0] == domain], ref + assert domain not in key, ref + keyed = sorted( + label + for walks in rule.values() + for label, (_, key) in walks.items() + if domain in key + ) + assert keyed == ["uk.spine-gates@1 assembled", "uk.stage.frs_relationships@1"] + assert domain in rule["uk.spine-gates@1"]["uk.spine-gates@1 assembled"].walk.reach + _, hashed, stage = _stage_walks()["frs_relationships"] + assert domain in stage.reach and domain in hashed + + +def test_every_reader_of_the_annuity_and_trust_column_hashes_its_module() -> None: + """The annuity and trust column's name lives in annuity_trust_column, + which holds that name alone, and every module that reads it imports it + from there (microcosm#931 ruling 4). The measure resolver reads zeros + where a frame lacks the column, so a renamed constant would change the + measures without re-keying them, and the name cannot be listed as + unbound. Every stage and kernel whose code reads it hashes the module: + the spine root, which writes the column; the SPI income stage, which + zeros it on the rows it draws, though it could chain on the root's key; + and the dense measures and the national problem, which compute through + nothing in frs_spine, so their keys leave it out + (test_uk_target_compile_modules holds both kernels' hashes to an edit to + either module).""" + from microcosm.build.uk_runtime import annuity_trust_column, frs_spine + + column = (annuity_trust_column.__name__, "ANNUITY_AND_TRUST_INCOME_COLUMN") + holder = _module_definitions(column[0], _HELPER_SCOPE).definitions + assert set(holder) == {column[1]} + assert _helper_kind(*column) == "name" + runtime = Path(frs_spine.__file__).parent + for path in sorted(runtime.glob("*.py")): + if path.stem == "annuity_trust_column": + continue + name = frs_spine.__package__ + if path.stem != "__init__": + name = f"{name}.{path.stem}" + facts = _module_definitions(name, _HELPER_SCOPE) + assert column[1] not in facts.definitions, name + assert facts.imports.get(column[1], column) == column, name + stages = { + stage: annuity_trust_column.__name__ in hashed + for stage, (_, hashed, walk) in _stage_walks().items() + if column in walk.stops + } + assert {"frs_spine", "hmrc_spi_income_spine"} <= set(stages), stages + assert all(stages.values()), stages + readers = { + label: (walk, key) + for walks in _kernel_rule_walks().values() + for label, (walk, key) in walks.items() + if column in walk.stops + } + assert {"uk.full.measures@1", "uk.full.national_problem@1"} <= set(readers) + for label, (_, key) in readers.items(): + assert annuity_trust_column.__name__ in key, label + for label in ("uk.full.measures@1", "uk.full.national_problem@1"): + walk, key = readers[label] + assert frs_spine.__name__ not in walk.reach, label + assert not [item for item in walk.stops if item[0] == frs_spine.__name__] + assert frs_spine.__name__ not in key, label + + def test_each_gate_battery_walks_the_engine_its_build_hands_it() -> None: """The engine a battery's phase walk follows (``_BATTERY_ENGINES``) is the one its build constructs the battery's kernel with, and the kernel diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index 1dbde579b9..2fb115ffaf 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -310,6 +310,51 @@ def test_an_edit_to_calibration_run_leaves_the_kernels_keys( assert kernel.implementation_hash() == before +@pytest.mark.parametrize( + "edited_module", + ("frs_spine", "frs_relationships"), +) +@pytest.mark.parametrize( + ("kernel", "module"), + _TARGET_AND_PROBLEM_KERNELS, + ids=[kernel.ref for kernel, _ in _TARGET_AND_PROBLEM_KERNELS], +) +def test_an_edit_to_frs_spine_or_frs_relationships_leaves_the_kernels_keys( + tmp_path, monkeypatch, kernel, module, edited_module +): + """No target or problem kernel hashes frs_spine or frs_relationships + (microcosm#931 ruling 4). The dense measures and the national problem + read nothing of frs_spine's but the annuity and trust column's name, + which lives in annuity_trust_column. The target compilation and the + national targets reach the ONS household type domain only through + ledger_targets' household-type check, which only refuses and is listed + refusal-only in test_uk_graph. So on a bound spine an edit to either file + re-keys none of these kernels, and the national line does not re-solve + for it. In the in-graph build the spine stages that hash them still + re-key every node after them.""" + edited = importlib.import_module(f"microcosm.build.uk_runtime.{edited_module}") + before = kernel.implementation_hash() + source = Path(edited.__file__) + copy = tmp_path / source.name + copy.write_bytes(source.read_bytes() + b"\n# an edited line\n") + monkeypatch.setattr(edited, "__file__", str(copy)) + assert kernel.implementation_hash() == before + + +@pytest.mark.parametrize("ref", ("uk.full.measures@1", "uk.full.national_problem@1")) +def test_an_edit_to_the_annuity_and_trust_column_rekeys_the_measures( + tmp_path, monkeypatch, ref +): + """The measure resolver reads zeros where a frame lacks the annuity and + trust column, so a renamed constant would change the measures without + re-keying them: the dense measures and the national problem hash the + column's module (microcosm#931 ruling 4).""" + from microcosm.build.uk_runtime import annuity_trust_column + + kernel, _ = _KERNELS_BY_REF[ref] + assert_an_edit_rekeys(kernel, annuity_trust_column, tmp_path, monkeypatch) + + def test_an_edit_to_calibration_run_leaves_the_preflight_and_the_solve( tmp_path, monkeypatch ): From 1fbb41fbd4ad61cb2ea7a8fb04678765509b3a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:39:52 +0000 Subject: [PATCH 051/108] Count module-level aliases and wrapper internals in the hold's premises test (#931) The identity tool's count hold rests on _SUPPORT_SPLIT_CORRECTED_ROWS = 3: the split, the incidence clone and the residential split each correct their total once. test_the_hold_counts_every_weight_change_after_the_split reads that premise from the stages' code. Run A's re-review of a0933a56d found it still passed a second correction in the incidence clone made through an import alias, through a module-level alias, or by a wrapper that corrects twice, since it counted references only inside function bodies and a wrapper call as one correction whatever the wrapper did. This commit takes that nit, the third. It edits the test only. - Aliases. A name one of the three stage modules binds outside every function, at module level or in a class body, to one of spi_support's two helpers, a wrapper or another alias is an alias, and a reference to it counts as a reference to a wrapper does. That covers a from-import of one (its asname is the alias) and an assignment whose value refers to one, a functools.partial among them. Wrappers and aliases are found to one fixed point over the three modules. A function-local import under another name now counts once, where it names the helper, as a local alias assignment already did. At the tip the scan finds no alias but the helpers' own from-imports. - Wrappers. The test pins the wrapper set to what the code has, cgt_support._exact_support_weights and cgt_residential_split._exact_residential_weights, and each to one reference to each of the two helpers: each tries _exact_total_correction on a household its split left whole and falls back to _importance_weights_with_exact_total. So a second correction inside a wrapper fails, and so does a new _exact... wrapper. Wrappers are keyed by module and qualified name. - Text. The comment over the count says which forms it catches, and that it counts references, not calls: a loop that corrects twice, or a local alias called twice, counts once. No docstring or changelog fragment said the test catches aliases in general; only that comment did. Correction to a0933a56d's message, for the PR body: its identity tool bullet said a second correction through an alias or a partial fails the premises test. Then only a local alias or partial inside the correcting function did; an import alias, a module-level alias or partial and a wrapper that corrects twice passed. Now all of them fail. What moves: nothing. No kernel hash, node key, value, declaration or generated file moves; source_stages.json and the coverage manifest are current (both --check pass at this commit). Tests (engine-free; code facts only): the touched file passes (101 tests). Run A's harness and mutations, copied to scratch and applied to a git archive snapshot of f274857e9 with this test laid over it: the unmutated snapshot passes, Pimport, Pmodule and Pwrapper2 now fail (all three pass at f274857e9), and the control Pbare still fails. Five more mutations fail here and pass at f274857e9: a top-level functools.partial alias, an alias of a module-level alias, a function-local import under another name, a class-body alias called from the clone transform's __call__, and a new _exact... wrapper that corrects twice. Run A's seven fix-round mutations still fail (an attribute call, a bare name in __call__, another module's wrapper, a local alias, a nested function, a lambda and a local partial). A loop running the clone's one correction twice passes, as the comment says. Ruff is clean on the edited file. Co-Authored-By: Claude Opus 5.5 --- .../uk/test_uk_identity_stability_receipts.py | 129 +++++++++++++++--- 1 file changed, 108 insertions(+), 21 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 509e905ae8..b66bdb249a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -2190,6 +2190,7 @@ def test_the_hold_counts_every_weight_change_after_the_split(self) -> None: """ import ast import inspect + from collections import Counter from microcosm.build.uk_runtime import ( cgt_residential_split, @@ -2227,14 +2228,28 @@ def test_the_hold_counts_every_weight_change_after_the_split(self) -> None: assert boundaries == {"uk.identity@1"} # Each reference that can correct a frame's total, by the function or - # method making it: to spi_support's helpers, or to a function of any - # of the three stages' modules that wraps them (one named _exact... - # that refers to a helper or another wrapper), by bare name or as an - # attribute (spi_support._importance_weights_with_exact_total(...), - # cgt_support._exact_support_weights(...)), called or not, so an - # alias or a partial of one counts too. Every function, method and - # nested function counts its own references, not those of the - # functions it nests. + # method making it. A reference is a bare name or an attribute + # (spi_support._importance_weights_with_exact_total(...), + # cgt_support._exact_support_weights(...)), called or not, to one of + # spi_support's two helpers, a wrapper or an alias: + # - a wrapper is a function of any of the three stages' modules named + # _exact... that refers to one; + # - an alias is a name one of those modules binds to one outside every + # function, at module level or in a class body: imported under + # another name (from ... import _importance_weights_with_exact_total + # as _total), or assigned from an expression that refers to one + # (_total = _importance_weights_with_exact_total, or a + # functools.partial of it). + # Wrappers and aliases are found to a fixed point over the three + # modules, so one module's wrapper or alias counts where another module + # refers to it, and a reference to an alias counts as a reference to a + # wrapper does. Every function, method and nested function counts its + # own references, not those of the functions it nests. A lambda's + # reference counts in the function that writes it, and a local alias, a + # local partial or a local import under another name counts once, where + # it names the helper or wrapper. What is counted is references, not + # calls: a loop that corrects twice, or a local alias called twice, + # counts once. helpers = {"_exact_total_correction", "_importance_weights_with_exact_total"} stages = ("cgt_support_split", *weighing) modules = {graph_kernels._stage_module(stage) for stage in stages} @@ -2256,7 +2271,8 @@ def collect(node, prefix: str, into: dict[str, ast.AST]) -> None: def referred(function) -> list[str]: """The names ``function`` refers to outside the functions it nests: - each bare name and each attribute read, a call's included.""" + each bare name and each attribute read, a call's included, and each + name it imports under another name.""" names: list[str] = [] pending = list(ast.iter_child_nodes(function)) @@ -2268,10 +2284,57 @@ def referred(function) -> list[str]: names.append(node.id) elif isinstance(node, ast.Attribute): names.append(node.attr) + elif isinstance(node, ast.alias) and node.asname is not None: + names.append(node.name) pending.extend(ast.iter_child_nodes(node)) return names + def named(node) -> set[str]: + """Each bare name and attribute name under ``node``, itself included.""" + + return { + item.id if isinstance(item, ast.Name) else item.attr + for item in ast.walk(node) + if isinstance(item, ast.Name | ast.Attribute) + } + + def outside_functions(node): + """The statements under ``node`` outside every function: at module + level, in a block there, or in a class body.""" + + for child in ast.iter_child_nodes(node): + if isinstance(child, defined): + continue + if isinstance(child, ast.stmt): + yield child + yield from outside_functions(child) + + def aliases(tree, counted: set[str]) -> set[str]: + """The names ``tree`` binds outside every function to a name in + ``counted``: imported under another name, or assigned from an + expression that refers to it (every name in the targets).""" + + bound: set[str] = set() + for node in outside_functions(tree): + if isinstance(node, ast.ImportFrom): + bound |= { + alias.asname or alias.name + for alias in node.names + if alias.name in counted + } + elif ( + isinstance(node, ast.Assign | ast.AnnAssign | ast.AugAssign) + and node.value is not None + and named(node.value) & counted + ): + targets = ( + node.targets if isinstance(node, ast.Assign) else [node.target] + ) + bound |= set().union(*(named(target) for target in targets)) + return bound + every: dict[str, dict[str, ast.AST]] = {} + trees: dict[str, ast.Module] = {} for module in modules: tree = ast.parse(inspect.getsource(module)) functions: dict[str, ast.AST] = {} @@ -2280,26 +2343,50 @@ def referred(function) -> list[str]: node.name for node in ast.walk(tree) if isinstance(node, defined) } == {function.name for function in functions.values()} every[module.__name__] = functions - # The wrappers of all three modules, to a fixed point, so a wrapper - # one module defines counts where another module calls it. - wrappers: set[str] = set() + trees[module.__name__] = tree + # The names a reference to which counts: the helpers, the wrappers' + # names and the aliases, to a fixed point over all three modules. + counted = set(helpers) while True: - more = { - function.name - for functions in every.values() - for function in functions.values() + wrappers = { + (module_name, name) + for module_name, functions in every.items() + for name, function in functions.items() if function.name.startswith("_exact") - and set(referred(function)) & (helpers | wrappers) + and set(referred(function)) & counted } - if more == wrappers: + more = ( + helpers + | {every[module_name][name].name for module_name, name in wrappers} + | set().union(*(aliases(tree, counted) for tree in trees.values())) + ) + if more == counted: break - wrappers = more + counted = more + # The wrappers are the support split's and the residential split's own + # exact-total corrections. Each tries _exact_total_correction on a + # household its split left whole (the support split on up to 64 in + # turn, taking the first that carries it) and falls back to + # _importance_weights_with_exact_total where none does. So each + # corrects once and refers once to each helper, and a second + # correction written into one fails here. + assert { + (module_name, name): Counter( + item for item in referred(every[module_name][name]) if item in counted + ) + for module_name, name in wrappers + } == { + (cgt_support.__name__, "_exact_support_weights"): Counter(helpers), + (cgt_residential_split.__name__, "_exact_residential_weights"): Counter( + helpers + ), + } corrections: dict[tuple[str, str], int] = {} for module_name, functions in every.items(): for name, function in functions.items(): - if function.name in wrappers: + if (module_name, name) in wrappers: continue - count = sum(item in helpers | wrappers for item in referred(function)) + count = sum(item in counted for item in referred(function)) if count: corrections[(module_name, name)] = count assert corrections == { From d3085cece92d05959732abbf408b15e5bc8fbb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:26:35 +0000 Subject: [PATCH 052/108] Follow up the review of the phase-code, target-key and premises-test commits (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three reviews of b289e8f91, f274857e9 and c7e0523ef found no must, two shoulds and fifteen nits, some of them the same finding twice. This commit takes the shoulds and the nits that need no ruling. The review's corrections to the earlier messages are below, for the PR body. It moves keys only: every edit to a hashed file is to a comment or docstring. Citations (the shoulds): - The texts for the 9 Nov (late) rulings cited them as microcosm#931 ruling 3, 3a, 3b and 4. In the PR body those numbers belong to the 9 Nov (evening) rulings, and the 9 Nov (late) block is unnumbered, so a reader following one would land on an unrelated ruling. The 24 sites now cite "microcosm#931, ruled 9 Nov (late)", as the changelog does: in gate_phase_codes, graph_evidence, graph_terminal, graph_kernels, graph_national, graph_targets, frs_spine, annuity_trust_column and four test files. "Ruling 6" still means the evening ruling. Phase codes: - The cache hit. Texts said the spine gates' new refusal means a declaration replayed from another process cannot key a spine gate on stale phase code. The check runs only when the kernel runs, and the executor serves a record stored under a node's key without calling run. UKSpineGateKernel's docstring, gate_phase_codes' module docstring, require_current_phase_code's docstring, the spine stale-code test's docstring and the changelog now say what holds: a node with a stale or missing _code is refused whenever its kernel runs, so no gate evaluates its phase under a key that names other phase code, and the check does not guard a cache hit. - test_each_gate_kernel_checks_its_phase_code_first also holds that no statement before the check returns or yields, outside the functions it defines, so every path through the run reaches the check or refuses first. It also holds that the run binds phase exactly once, from its node's parameter, before the check, so the phase evaluated is the phase checked. A return in a parameter branch before the check, and phase rebound after it, passed before and fail now, in either battery. - A new test, test_the_phase_code_check_refuses_what_its_tables_leave_out, calls the check directly. A phase its battery's table leaves out passes with no _code and is refused with one, whatever its value, and a battery with no table is refused. No gate node reaches either branch today. - test_no_uk_graph_kernel_hashes_the_phase_codes' docstring said an edit to what one phase computes through re-keys no node whose kernel hashes graph_evidence or graph_terminal. It now says what the test holds: a list edit in gate_phase_codes moves that phase's parameter, which re-keys that phase's node and every node after it, and moves no kernel's hash. - The changelog's phase-codes bullet now gives what a list edit in graph_evidence or graph_terminal used to re-key, chaining included. For a spine phase's list that was every node of the spine and of the in-graph build, and all but the checkpoint of the local line on a bound spine. For a full phase's list it was the preflight, the dense solve and every node after them. Simulated edits to the two files at c7e0523ef, where they have the same hashers as at 4df40658e, re-key 113 of 113, 32 of 33 and 0 of 6 nodes for the first and 9, 9 and 0 for the second (in-graph build, local line on a bound spine, national line). The What moves sentences for the phase codes and for the column now add "and by key chaining every node after one". Target keys: - frs_spine's PENTYPE comment said the nodes that read the column's name key on it without keying on frs_spine. annuity_trust_column's docstring said the same of the target and problem kernels. Both now name the two kernels that do: the dense measures and the national problem. The create kernel and the SPI income stage read the name and hash both modules, and the other four target and problem kernels neither read nor hash it. The docstring also says an edit to frs_spine would otherwise re-solve both lines on a bound spine, not only the national one. - test_every_reader_of_the_annuity_and_trust_column_hashes_its_module now pins frs_spine as the household-type test pins frs_relationships: every key that covers frs_spine is the create kernel's or a spine stage's. Neither runs on the local or the national line on a bound spine, so the test now holds the changelog's claim that an edit to frs_spine re-keys no node of either line there. It also refuses a uk_runtime module reading the column's name as an attribute of any module but annuity_trust_column. measure_simulation reading it through frs_spine, which imports it, passed every target test before. - The changelog's column bullet names both checks, and its gate-battery bullet names the new phase-code checks. Premises test: - The alias scan also reads spi_support's bindings outside every function, since spi_support defines the helpers. So an alias spi_support binds, plain or as a functools.partial, which the clone imports and calls for a second correction, now counts. spi_support's functions are not counted as correction sites, and at the tip the scan finds nothing there. - The comment says the limits. Wrappers are looked for only in the three stage modules. A correction made through another module's function that corrects a total itself, such as spi_band_donors' _exact_incumbent_total or the spi_support functions that call the fallback, is not counted. Nor is one made through a helper looked up by a name built at run time. - The review's other fix was a fixed point over every uk_runtime module. It is not taken, and the test states the limit instead, which the review offered as the alternative. By name, that fixed point takes in 18 functions across 7 modules at the tip, the stage transforms' __call__ methods among them. The count would then have to tell correction sites from their callers, and keying it by module would need import resolution in the test. Not taken, for María: the check does not guard a cache hit. Closing that would need a check before keying, for example the drivers comparing each gate node's declared _code with gate_phase_codes before run_graph. No production path replays a stored declaration today, since full_build_cli and spine_build build their graphs in-process. Corrections to the earlier messages, for the PR body: - b289e8f91, the first What moves bullet. Its per-file counts (65, 8, 5 and 2) overlap. Split disjointly: 65 kernels hash graph_evidence.py; 5 more hash graph_terminal.py (the four export kernels and the rotated holdout); 2 more hash gate_scopes.py (the bound spine and the national gates); none more hash calibration_run.py. That makes 72. - c7e0523ef, the correction to a0933a56d. It should read: an import alias, a module-level alias or partial, and a wrapper that refers to a helper a second time all passed then; all three fail now, as does an alias spi_support binds (this commit). A correction repeated through one reference, in a loop, still passes, as the test's comment says. So does one made through another module's own corrector. What moves: node keys only. No value, declaration or generated file moves, and neither the solve nor the country fingerprint does. - 80 of the 99 UK kernel hashes move: exactly those that hash a file this commit edits. Split disjointly: - 57 hash graph_kernels.py: the spine stage, create, claim and identity kernels, and the spine gates; - 10 more hash graph_evidence.py: the preflight and calibrated gates, the sample, normalize, expand, pool and locations kernels, the geography mapping and gate, and the certification readiness; - 2 more hash annuity_trust_column.py: the measures and the national problem; - 6 more hash graph_targets.py: the compilation, the selection, the problem, the rotated holdout, and the national targets and gates; - 1 more hashes graph_national.py: the national readback; - 4 more hash graph_terminal.py: the export kernels. - frs_spine.py's 20 hashers are all spine kernels counted above. No kernel's set of hashed modules changes, and none hashes gate_phase_codes. The other 19 keep theirs: the bound spine, the dense solve, the calibrated, selected and full identity kernels, the size checkpoint import, draw, refit and search, the geography, property and private rent kernels, and the four atomic geography kernels. No transport module imports a uk_runtime module, and none of the 31 transport and atomic geography kernels the key probe registers moves. No phase code moves. - Node keys: every node of the hermetic and gated spines (79 and 81) and of the in-graph build (113) re-keys. On a bound spine the local line re-keys 32 of 33 nodes and the national line 5 of 6; both keep only the checkpoint, whose kernel hashes none of the seven files. b289e8f91 already re-keys every node of every UK graph against 4df40658e, so the push re-keys nothing more. - source_stages.json and the coverage manifest are current; both --check pass at this commit. uk_spine.json and the H2 fixture hold stage contract hashes, so neither regenerates. Tests (engine-free; code facts and synthetic values): the five touched files and the shared rule and transport rule tests pass (286 tests). Ruff is clean on the 13 edited Python files. On a scratch copy of c7e0523ef with this commit laid over it, 17 mutations each do what is expected: - the unmutated copy passes; - run A's Pimport, Pmodule and Pwrapper2 fail the premises test, and so does the control Pbare; - an alias, or a top-level functools.partial, that spi_support binds and the clone calls fails it too; both pass at c7e0523ef; - the stated limits pass, as the comment says: a correction through spi_band_donors' _exact_incumbent_total, and one through a helper looked up by a built name; - a return before the check, or phase rebound after it, fails the static test alone, in the spine gates and in the full gates; - the check accepting any code on a phase outside its table, or an unknown battery getting past it, fails the new test alone; - measure_simulation reading the column through frs_spine or spi_spine, or the bound spine hashing frs_spine, fails the reader test alone. Co-Authored-By: Claude Opus 5.5 --- changelog.d/931-uk-helper-bindings.changed.md | 6 +- .../build/uk_runtime/annuity_trust_column.py | 7 +- .../microcosm/build/uk_runtime/frs_spine.py | 4 +- .../build/uk_runtime/gate_phase_codes.py | 36 +++-- .../build/uk_runtime/graph_evidence.py | 13 +- .../build/uk_runtime/graph_kernels.py | 5 +- .../build/uk_runtime/graph_national.py | 4 +- .../build/uk_runtime/graph_targets.py | 6 +- .../build/uk_runtime/graph_terminal.py | 2 +- .../tests/engine_free/uk/test_uk_graph.py | 151 +++++++++++++++--- .../engine_free/uk/test_uk_graph_evidence.py | 11 +- .../engine_free/uk/test_uk_graph_terminal.py | 2 +- .../uk/test_uk_identity_stability_receipts.py | 28 ++-- .../uk/test_uk_target_compile_modules.py | 8 +- 14 files changed, 207 insertions(+), 76 deletions(-) diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 663efad862..2d92e5452e 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -62,14 +62,14 @@ Every UK spine stage now hashes each first-party module its transform computes t - **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Where a phase's gates read the engine their kernel hands them, the walk also follows the methods they call on it, from the engine's class, as for an engine stage (see The engine): the spine gates' rules engine is the spine build's `PolicyEngineUKEngine`, and the full gates' engine is the driver's coverage engine. Another test checks each part of that model against the code, the engines among them. So the assembled gate's own key, which the spine stages after it chain on, carries nothing only the transferred phase computes through, and the preflight's, which the dense solve chains on, nothing only the terminal phase does. Key chaining still re-keys them wherever a node upstream hashes the edited module. Of the 17 modules only the transferred phase computes through, stages at or before `frs_brma`, the assembled phase's last, hash `stochastic_assignment` and `take_up_contract`, so an edit to either re-keys the assembled gate; no stage before it hashes the other 15. Of the 38 only the terminal phase computes through, most re-solve too, through the spine stages, the target and problem kernels, the population kernels or the dense solve, which hash them as well. Eight never re-solve, on any line: `high_value_home_gates`, `microcosm.diagnostics`' catalogs and schema, `calibrate.hierarchy_labels`, the target loss attribution, `input_mass`, `cgt_projection` and `source_runtime`, so a disclosure fix between two builds that must reuse the solve still belongs in `high_value_home_gates`. `uk_runtime.diagnostics` re-solves only the national line, whose problem kernel hashes it. `calibration_run` and `release_certification` re-solve only on a bound spine, whose kernel hashes both, and seven modules that only the spine's nodes hash before the solve re-solve only in the in-graph build. The preflight's own code, the shared ledger's parity report, is also hashed by the full build's target compilation, selection, measures and problem, by the national line's targets, problem and gates and, in the in-graph build, by the `etb_services` and `lcfs_consumption` stages and the transferred gate, so an edit to it re-keys those too. - The spine gates' hash binds the 12 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants`, `calibrate.geography_constants` and the engine adapter, through whose `PolicyEngineUKEngine.enum_domain` each phase's enum gates resolve a domain no artifact supplies, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. The spine gate kernel refuses a stale or missing `_code`, as the full gates do (see The phase codes, below). - The full gates' hash binds the 26 modules both phases compute through. It loses `local_rowwise`, `weighted_integrity` and `diagnostics`, which only the terminal phase computes through. It binds the spine and local gate scopes and the scoped gate manifest, which the preflight computes through, in a new `gate_scopes` module, not in `calibration_run`, the national calibration runner they lived in. They moved there unchanged, and `calibration_run` re-exports them, as microcosm#1492 moved the target kernels' two functions into `ledger_surface`, so in the in-graph build an edit elsewhere in the runner re-keys neither the preflight nor the solve. The bound spine, the national gates and the certification, whose code reaches them too, hash `gate_scopes` as well. The calibrated gates' `terminal_code` binds `calibration_run` itself, for the admin totals and CGT projection artifact their gate context is built with. The preflight's release input coverage gate computes through `stage_evidence`, `parity_reference`, `release_identity` and `release_input_coverage`, which its key used to miss, so on a bound spine an edit to any of them now re-keys the preflight and re-solves; in the in-graph build so does one to `parity_reference` or `release_identity`, while the other two already re-solved through the spine stages that hash them. The preflight node carries a new `preflight_code`: the shared ledger's registry parity, behind the ledger compile parity gates only the preflight declares. The calibrated node's `terminal_code` now hashes 38 modules. `high_value_homes` and `calibrate.geography_constants` move into the shared hash, since the preflight computes through them too; the target kernels hash both, so an edit to either already re-solved. The kernel refuses a stale or missing `preflight_code`, as it refuses a stale `terminal_code`. - - The phase codes. As María ruled on 9 Nov (late), the four functions that compute them and their two tables live in a small module of their own, `uk_runtime.gate_phase_codes`, unchanged in what they hash, with one stale-code check, `require_current_phase_code`, which both batteries' kernels call before they read anything of their node but its parameters. In `graph_evidence` and `graph_terminal`, which define the gate kernels and which every spine stage kernel, the dense population kernels and the export kernels hash too, an edit to a phase's list of modules, which the battery test demands whenever a phase's evaluators gain one, re-keyed every node whose kernel hashes either file: the whole spine, or the preflight and the solve. Now only the graph construction, which sets each node's `_code`, and the check read the module, and no kernel hashes it: the check is listed refusal-only, since it reads its parameters and the phase's code and refuses, writing nothing, and a phase's own code reaches its key through the parameter, which `node_key` hashes. So an edit to a phase's list moves only that phase's parameter, which re-keys that phase's node and what follows it. `source_hash` leaves the kernels' listed core names: only the full gates' run named it, through the phase-code table, which only the check reads now. The spine gates' run now refuses a stale or missing `_code` too, a new refusal, so a graph declaration replayed from another process cannot key a spine gate on stale phase code. Tests check that no UK kernel's key covers the module, that both gate kernels call the check first, on their own battery's table, and that each spine phase refuses a stale and a missing code and passes its node's own. + - The phase codes. As María ruled on 9 Nov (late), the four functions that compute them and their two tables live in a small module of their own, `uk_runtime.gate_phase_codes`, unchanged in what they hash, with one stale-code check, `require_current_phase_code`, which both batteries' kernels call before they read anything of their node but its parameters. In `graph_evidence` and `graph_terminal`, which define the gate kernels and which every spine stage kernel, the dense population kernels and the export kernels hash too, an edit to a phase's list of modules, which the battery test demands whenever a phase's evaluators gain one, re-keyed every node whose kernel hashes either file, and by key chaining every node after one: for a spine phase's list every node of the spine and of the in-graph build and all but the checkpoint of the local line on a bound spine, for a full phase's list the preflight, the dense solve and every node after them. Now only the graph construction, which sets each node's `_code`, and the check read the module, and no kernel hashes it: the check is listed refusal-only, since it reads its parameters and the phase's code and refuses, writing nothing, and a phase's own code reaches its key through the parameter, which `node_key` hashes. So an edit to a phase's list moves only that phase's parameter, which re-keys that phase's node and what follows it. `source_hash` leaves the kernels' listed core names: only the full gates' run named it, through the phase-code table, which only the check reads now. The spine gates' run now refuses a stale or missing `_code` too, a new refusal, so no gate evaluates its phase under a key that names other phase code. A record stored under a gate node's key is still reused without running its kernel, so the check does not guard a cache hit. Tests check that no UK kernel's key covers the module; that both gate kernels call the check first, on their own battery's table, with no return before it and their phase bound once, before it; that each spine phase refuses a stale and a missing code and passes its node's own; and that the check refuses a code on a phase its battery's table leaves out, and a battery with no table. - The national gates evaluate whichever gates their node's scope names, so their hash binds everything the gate registry's bindings compute through. That is 50 modules, which `battery_bindings.UK_GATE_REGISTRY_MODULES` lists and a test holds exactly to the registry's derived reach. Their stop at the registry, the one held kernel stop, goes. - **Target and problem kernels.** As María ruled on 9 Nov, the derived rule replaces microcosm#1497's per-kernel import record for the full build's target compilation, selection, measures and problem and the national targets and problem, which that record held until now. No kernel stays held. - `test_uk_target_compile_modules` loses the record (`UNCLASSIFIED_IMPORTS`), its kernel groups and the check that read them. It keeps the shared list's closure check, the tests of what the kernels hash and its hash inputs, the seven above among them. A new check covers what the derived rule does not: each hash input is still imported where listed, and a kernel hashes one only where another module it hashes imports it too. So no kernel hashes the compile list or the solve fingerprint's module, and an edit to either re-keys none. - What all four full kernels compute through goes in the hash they share, and what only one computes through goes in its own implementation hash, as for the size search and refit. The shared hash gains the canonical encoder. The compilation adds `chronicle_epoch`, whose schema versions and fact-key epochs `ledger_artifact` checks the facts against and records. The measures add the 15 modules that resolving the measures and compiling their national problem compute through: `graph_population` and `national_frame`, for the frame they rebuild; `atomic_area_support`, `frame.materialize` and the engine adapter, for the engine's scratch frame; `dfe_spring_term`, `frs_annual_income`, `hmrc_income`, `income_tax_rate_classes`, `lcfs_consumption` and `uc_target_measurements`, for the resolver's inputs and measures; `chronicle_epoch`, for the vendored facts' schema checks; `national_calibration`, for the measure inputs it injects; `local_rowwise`, for the national rows; and `calibrate.monetary_binding`, for the constraint matrix's check. The problem adds `graph_population`, `national_frame`, `local_geography` (the local surface matrix's area columns) and `calibrate.monetary_binding`. - The national targets' and problem's shared hash gains `chronicle_epoch`, `graph_targets` (the register codec) and the canonical encoder. The national targets compute through nothing else, so they have no hash of their own. The national problem adds the measures' modules but `chronicle_epoch` and `national_calibration`, which its shared hash already binds, and adds `diagnostics` and `gate_battery`, whose target geography levels and JSON-safe values its payload records. - - The measures and the national problem also bind the annuity and trust column's name. The resolver reads that column where the frame carries it and reads zeros where it does not, so on the national line's bound spine a renamed constant would change the problem without re-keying it, and the name cannot be listed as unbound. As María ruled on 9 Nov (late), the name moved unchanged from `frs_spine` into a module of its own, `uk_runtime.annuity_trust_column`, which holds nothing else, and `frs_spine`, `measure_simulation`, `spi_spine` and `spi_income` each import it from there. So the two kernels bind that module, not `frs_spine`, whose every edit would otherwise re-key them. The `frs_spine` root, which writes the column, binds it too, and so does `hmrc_spi_income_spine`, which zeros it on the rows it draws, though that stage could chain on the root's key; a test holds every stage and kernel whose code reads the name to its module. The clone index columns' name stays listed for them: the dense measures read it only to split their frame into the engine blocks they are asked for, and refuse a frame without it, and their population is the expand's; the national problem resolves its measures in one block, which never reads it. A test checks these facts. + - The measures and the national problem also bind the annuity and trust column's name. The resolver reads that column where the frame carries it and reads zeros where it does not, so on the national line's bound spine a renamed constant would change the problem without re-keying it, and the name cannot be listed as unbound. As María ruled on 9 Nov (late), the name moved unchanged from `frs_spine` into a module of its own, `uk_runtime.annuity_trust_column`, which holds nothing else, and `frs_spine`, `measure_simulation`, `spi_spine` and `spi_income` each import it from there. So the two kernels bind that module, not `frs_spine`, whose every edit would otherwise re-key them. The `frs_spine` root, which writes the column, binds it too, and so does `hmrc_spi_income_spine`, which zeros it on the rows it draws, though that stage could chain on the root's key; a test holds every stage and kernel whose code reads the name to its module, and every module that reads the name to reading it from there, not through another module that imports it. The same test holds every key that covers `frs_spine` to the create kernel and the spine stages, none of which runs on the local or the national line on a bound spine. The clone index columns' name stays listed for them: the dense measures read it only to split their frame into the engine blocks they are asked for, and refuse a frame without it, and their population is the expand's; the national problem resolves its measures in one block, which never reads it. A test checks these facts. - `ledger_targets`' household-type check is listed refusal-only, as María ruled on 9 Nov (late). It compares the contract's `ons_household_type` bindings with the ONS household type domain and refuses a binding outside it or composition rows that do not cover it once each. It writes only into the dict and set it creates, which the shared refusal check counts as local. Its one caller, `compile_uk_target_registry`, calls it before the loop that compiles the references, on the compilation's and the national targets' paths alike. So neither kernel binds `frs_relationships`, which the rule asked of them only through this check. A test checks the check's place in its caller, that neither kernel computes through `frs_relationships` another way, and that the kernels that hash it for their own reasons keep it: the `frs_relationships` stage, whose transform it defines, and the assembled spine gate, whose own code holds the domain its enum gate checks. - No kernel drops a module it hashed before, though each still hashes some modules its walk does not reach. The full kernels' shared hash still binds the whole compile list and the modules beside it in all four, so the selection's key covers 51 modules while its code computes through 6. - `transport/compile_modules.py`'s reason for leaving the graph runtime off the transport compile list said that no kernel in the repository hashes a `microcosm.graph` module, which was untrue. It now says what the listed modules use those modules for, that the compile kernels hash none of them, and that other kernels hash the canonical encoder, each UK graph kernel whose code computes through it among them. No kernel hashes that file, so the edit re-keys nothing. -- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. The target and problem kernels' keys move once with their new bindings. `graph_targets.py` and `graph_national.py`, which define them, changed with them, and the preflight and calibrated gates, the rotated holdout and the national gates and readback each hash one or both of those files too. So in the full build the target compilation, selection, measures and problem, the pool checkpoint (`uk.full.pool`), the preflight, the rotated holdout, and the dense solve and every node after it re-key, and on the national line every node after its bound spine root; no spine node and no transport node re-keys with them. The phase codes' move re-keys once more every node whose kernel hashes `graph_evidence.py`, `graph_terminal.py`, `gate_scopes.py` or `calibration_run.py`, which is every node of every UK graph, the bound spine of the local and national lines included, whose kernel hashes the last two; of the phase codes only `terminal_code` moves, since it hashes `calibration_run`. The column's move and the listed household-type check re-key once more every node whose kernel hashes a file they edit (`frs_spine.py`, `graph_kernels.py`, `graph_targets.py`, `graph_national.py`, `measure_simulation.py`, `spi_spine.py` or `spi_income.py`): every spine node and every node of the in-graph build, 26 of the local line's 33 nodes on a bound spine, and the national line's every node after its bound spine root. After them, on a bound spine an edit to `frs_spine.py` or `frs_relationships.py` re-keys no node of the local or the national line, where it re-keyed 12 and 4 of their nodes for `frs_spine.py` and 14 and 5 for `frs_relationships.py`, so neither line re-solves for one; in the in-graph build either still re-keys its stage and every node after it, as before. An edit to `annuity_trust_column.py` re-keys the spine root, the measures and the national problem, and what follows them. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. The 16 engine stages also hash `microcosm.frame.materialize`, and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. The target and problem kernels' keys move once with their new bindings. `graph_targets.py` and `graph_national.py`, which define them, changed with them, and the preflight and calibrated gates, the rotated holdout and the national gates and readback each hash one or both of those files too. So in the full build the target compilation, selection, measures and problem, the pool checkpoint (`uk.full.pool`), the preflight, the rotated holdout, and the dense solve and every node after it re-key, and on the national line every node after its bound spine root; no spine node and no transport node re-keys with them. The phase codes' move re-keys once more every node whose kernel hashes `graph_evidence.py`, `graph_terminal.py`, `gate_scopes.py` or `calibration_run.py`, and by key chaining every node after one: every node of every UK graph, the local and national lines on a bound spine from their root, whose kernel hashes the last two; of the phase codes only `terminal_code` moves, since it hashes `calibration_run`. The column's move and the listed household-type check re-key once more every node whose kernel hashes a file they edit (`frs_spine.py`, `graph_kernels.py`, `graph_targets.py`, `graph_national.py`, `measure_simulation.py`, `spi_spine.py` or `spi_income.py`), and by key chaining every node after one: every spine node and every node of the in-graph build, 26 of the local line's 33 nodes on a bound spine, and the national line's every node after its bound spine root. After them, on a bound spine an edit to `frs_spine.py` or `frs_relationships.py` re-keys no node of the local or the national line, where it re-keyed 12 and 4 of their nodes for `frs_spine.py` and 14 and 5 for `frs_relationships.py`, so neither line re-solves for one; in the in-graph build either still re-keys its stage and every node after it, as before. An edit to `annuity_trust_column.py` re-keys the spine root, the measures and the national problem, and what follows them. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py index 06f3b2a613..32c0ba4fa1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py @@ -12,9 +12,10 @@ The resolver reads zeros where a frame lacks the column, so a renamed constant would change the measures without re-keying them, and every node whose code reads the name binds this module. It holds the name alone, nothing that -computes, so the target and problem kernels key on the name without keying on -frs_spine, every edit to which would otherwise re-solve the national line on a -bound spine (microcosm#931 ruling 4). +computes, so the dense measures and the national problem key on the name +without keying on frs_spine, every edit to which would otherwise re-solve the +local and the national line on a bound spine (microcosm#931, ruled 9 Nov +(late)). """ from __future__ import annotations diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 334c6de566..9c3f0382d1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -442,8 +442,8 @@ def benefits_by_recipient( #: ANNUITY_AND_TRUST_INCOME_COLUMN, a calibration-only descriptor that the #: Table 3.7 other-income measure adds (microcosm#1428 review, finding 1). The #: descriptor's name lives in annuity_trust_column, which holds nothing else, -#: so the nodes that read it key on it without keying on this module -#: (microcosm#931 ruling 4). +#: so the dense measures and the national problem, which read it, key on it +#: without keying on this module (microcosm#931, ruled 9 Nov (late)). FRS_ANNUITY_AND_TRUST_PENSION_TYPES = (4, 5) #: Each person's company car from the FRS job tab, the four carriers the #: company_cars stage reads to set policyengine-uk's company car inputs diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py index 6688210afd..5917eb0f4a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py @@ -12,15 +12,18 @@ ``graph_terminal``, which define the gate kernels and which every spine stage kernel, the dense population kernels and the export kernels hash too, so an edit to a phase's list of modules re-keyed every node whose kernel -hashes either file. They moved here unchanged in what they hash, with the -one check both batteries call (microcosm#931 ruling 3a). Only the graph -construction, which sets each node's parameter, and that check read this -module. The check is listed refusal-only in test_uk_graph, and no kernel -hashes this module: a phase's own code reaches its key through the -parameter, which node_key hashes, and both gate kernels' runs call -:func:`require_current_phase_code` before they evaluate anything, so a node -whose parameter differs from the code this process computes is refused -(microcosm#931 ruling 3b, for the spine gates). +hashes either file, and every node after one. They moved here unchanged in +what they hash, with the one check both batteries call (microcosm#931, +ruled 9 Nov (late)). Only the graph construction, which sets each node's +parameter, and that check read this module. The check is listed +refusal-only in test_uk_graph, and no kernel hashes this module: a phase's +own code reaches its key through the parameter, which node_key hashes, and +both gate kernels' runs call :func:`require_current_phase_code` before they +evaluate anything, so a node whose parameter differs from the code this +process computes is refused whenever its kernel runs; the spine gates' +refusal was ruled on 9 Nov (late) too. A record stored under the node's key +is reused without running the kernel, so the check does not guard a cache +hit. """ from __future__ import annotations @@ -322,17 +325,20 @@ def require_current_phase_code( params: Mapping[str, object], phase: str, *, battery: str ) -> None: """Refuse a gate node unless its ``_code`` parameter is its phase's - own code as this process computes it (microcosm#931 ruling 3). + own code as this process computes it (microcosm#931, ruled 9 Nov (late)). ``battery`` is ``"spine"`` or ``"full"``, the battery whose table holds the phase's code (:data:`UK_SPINE_GATE_PHASE_CODE`, :data:`UK_FULL_GATE_PHASE_CODE`); a phase its table leaves out computes through nothing of its own, and its node must carry no parameter. The - graph construction sets the parameter and node_key hashes it, so a node - declared in another process, or replayed from a stored graph, with a stale - or missing parameter is refused before its kernel evaluates anything. The - check reads only its arguments and the phase's code, and writes nothing, - so it cannot change what its caller computes: test_uk_graph lists it as + graph construction sets the parameter, and node_key hashes it. A node + with a stale or missing parameter, one declared in another process for + instance, is refused whenever its kernel runs, before the kernel + evaluates anything, so no gate evaluates its phase under a key that names + other phase code. A record stored under the node's key is reused without + running the kernel, so the check does not guard a cache hit. The check + reads only its arguments and the phase's code, and writes nothing, so it + cannot change what its caller computes: test_uk_graph lists it as refusal-only, and no gate kernel's hash binds this module. """ tables = {"spine": UK_SPINE_GATE_PHASE_CODE, "full": UK_FULL_GATE_PHASE_CODE} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py index c77f2e497b..4cbb788981 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -358,9 +358,12 @@ class UKSpineGateKernel(KernelBase): module, as for ``stochastic_assignment`` and ``take_up_contract``. The kernel refuses a stale or missing ``_code`` before it evaluates anything, as the full gates do - (:func:`.gate_phase_codes.require_current_phase_code`; microcosm#931 - ruling 3b), so a graph declaration replayed from another process cannot - key a spine gate on stale phase code. + (:func:`.gate_phase_codes.require_current_phase_code`; microcosm#931, + ruled 9 Nov (late)), so a spine gate node whose ``_code`` is stale + or missing is refused whenever it runs, and no spine gate evaluates its + phase under a key that names other phase code. A record stored under the + node's key is reused without running the kernel, so the check does not + guard a cache hit. """ ref = "uk.spine-gates@1" @@ -454,8 +457,8 @@ def run(self, context: KernelContext) -> KernelResult: ) # The phase's own code keys the node through its parameter, not this # kernel's hash, so a node whose parameter is stale or missing is - # refused before anything is read or evaluated (microcosm#931 ruling - # 3b), as the full gates refuse theirs. + # refused before anything is read or evaluated (microcosm#931, ruled + # 9 Nov (late)), as the full gates refuse theirs. require_current_phase_code(context.params, phase, battery="spine") release_candidate = bool(context.params["release_candidate"]) synthetic_smoke = bool(context.params["synthetic_smoke"]) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index d70e952635..c0b642e848 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -339,7 +339,8 @@ # annual income descriptors on frs_annual_income's share columns, and the # evidence shares iterate HMRC's distributional inputs. It zeros a drawn # row's annuity and trust descriptor, whose name it reads from - # annuity_trust_column, as the root does (microcosm#931 ruling 4). + # annuity_trust_column, as the root does (microcosm#931, ruled 9 Nov + # (late)). "hmrc_spi_income_spine": ( spi_income, spi_rank_transport, @@ -375,7 +376,7 @@ # trace's digest, ages people by birth month, caps pension contributions # with their receipt, and codes council tax bands in the shared band order # (microcosm#931). It writes the annuity and trust descriptor under the - # name annuity_trust_column holds (microcosm#931 ruling 4). + # name annuity_trust_column holds (microcosm#931, ruled 9 Nov (late)). "frs_spine": ( uc_relationships, annuity_trust_column, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 6573d09929..bdeb66ed25 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -537,7 +537,7 @@ class UKNationalTargetKernel(_NationalKernel): # (frs_relationships) before it compiles. That check only refuses, so # test_uk_graph lists it refusal-only, and this kernel computes through # nothing its shared hash leaves out: it needs no hash of its own - # (microcosm#931 ruling 4). + # (microcosm#931, ruled 9 Nov (late)). ref = "uk.full.national_targets@1" def run(self, context: KernelContext) -> KernelResult: @@ -726,7 +726,7 @@ def implementation_hash(self) -> str: # since the resolver reads zeros where a frame lacks the column. The # name lives in annuity_trust_column, which holds nothing else, so # this hash leaves frs_spine out, and an edit there does not re-solve - # the national line (microcosm#931 ruling 4). + # the national line (microcosm#931, ruled 9 Nov (late)). return hashlib.sha256( canonical_json( { diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index a3a88414e4..e83b78397f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -290,8 +290,8 @@ def implementation_hash(self) -> str: # does not compute through the ONS household type domain # (frs_relationships): ledger_targets checks the household-type # bindings against it before it compiles, and that check only - # refuses, so test_uk_graph lists it refusal-only (microcosm#931 - # ruling 4). + # refuses, so test_uk_graph lists it refusal-only (microcosm#931, + # ruled 9 Nov (late)). return hashlib.sha256( canonical_json( { @@ -532,7 +532,7 @@ def implementation_hash(self) -> str: # It lives in annuity_trust_column, which holds nothing else, not in # frs_spine, which writes the column, so this hash leaves frs_spine # out, and on a bound spine an edit there leaves the measures' key - # alone (microcosm#931 ruling 4). + # alone (microcosm#931, ruled 9 Nov (late)). return hashlib.sha256( canonical_json( { diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 85cdcd706c..dc256e0efa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -829,7 +829,7 @@ class UKFullGateKernel(KernelBase): checks before it evaluates anything (:data:`.gate_phase_codes.UK_FULL_GATE_PHASE_CODE`, :func:`.gate_phase_codes.require_current_phase_code`; microcosm#931 - rulings 3 and 6). + ruling 6, and the module as ruled on 9 Nov (late)). """ ref = "uk.full-gates@1" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 0f309e527f..db41b23592 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -1197,7 +1197,7 @@ def changed_module_bytes(path): "hashes, so no gate kernel's hash binds the module " "(test_no_uk_graph_kernel_hashes_the_phase_codes), and an edit that " "changes what source_hash returns changes the parameter and the check " - "alike (microcosm#931 ruling 3)" + "alike (microcosm#931, ruled 9 Nov (late))" ), ( "microcosm.build.uk_runtime.ledger_targets", @@ -1212,7 +1212,7 @@ def changed_module_bytes(path): "and the full build's target compilation and the national targets " "both reach it there, so neither computes through frs_relationships " "(test_the_household_type_check_refuses_before_the_targets_compile; " - "microcosm#931 ruling 4)" + "microcosm#931, ruled 9 Nov (late))" ), } #: Names a stage reads from a module that neither its node nor any node @@ -2295,19 +2295,53 @@ def test_each_gate_battery_phase_keys_on_its_own_reach() -> None: def test_each_gate_kernel_checks_its_phase_code_first() -> None: """Each gate battery's kernel refuses a stale or missing ``_code`` before it reads anything of its node but its parameters, as the stale-code - check's refusal-only reason says (microcosm#931 ruling 3): its run calls - gate_phase_codes.require_current_phase_code in a statement of its own, on - its node's parameters, the phase it evaluates and its own battery's table, - and no statement before that reads the context beyond its parameters (the - node's artifacts, sources and population). So the spine gates refuse a - stale code as the full gates do. A kernel that read a phase-code table - outside the check would compute through gate_phase_codes, which the - kernel rule's test would find unhashed, since the rule does not follow - the listed check.""" + check's refusal-only reason says (microcosm#931, ruled 9 Nov (late)). Its + run calls gate_phase_codes.require_current_phase_code in a statement of + its own, on its node's parameters, the phase it evaluates and its own + battery's table. No statement before that reads the context beyond its + parameters (the node's artifacts, sources and population), returns or + yields, so every path through the run reaches the check or refuses + before it. The run binds ``phase`` once, from its node's parameter, + before the check, and never again, so the phase it evaluates is the phase + it checked. So the spine gates refuse a stale code as the full gates do. + A kernel that read a phase-code table outside the check would compute + through gate_phase_codes, which the kernel rule's test would find + unhashed, since the rule does not follow the listed check.""" import ast from microcosm.build.uk_runtime import gate_phase_codes + nested = ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda | ast.ClassDef + + def exits(node) -> bool: + """Whether ``node`` returns or yields, outside the functions, lambdas + and classes it defines.""" + + pending = [node] + while pending: + item = pending.pop() + if isinstance(item, ast.Return | ast.Yield | ast.YieldFrom): + return True + pending.extend( + child + for child in ast.iter_child_nodes(item) + if not isinstance(child, nested) + ) + return False + + def binds_phase(node) -> bool: + """Whether ``node`` binds or deletes the name ``phase``: an assignment + or loop target, an augmented or annotated assignment, a ``with`` or + walrus target, an exception handler, an import or a match capture.""" + + if isinstance(node, ast.Name): + return node.id == "phase" and isinstance(node.ctx, ast.Store | ast.Del) + if isinstance(node, ast.ExceptHandler | ast.MatchAs | ast.MatchStar): + return node.name == "phase" + if isinstance(node, ast.alias): + return (node.asname or node.name.partition(".")[0]) == "phase" + return False + check = (gate_phase_codes.__name__, "require_current_phase_code") assert check in _REFUSAL_ONLY_HELPERS kernels = _uk_graph_kernels() @@ -2344,7 +2378,16 @@ def test_each_gate_kernel_checks_its_phase_code_first() -> None: for sub in ast.walk(item) if isinstance(sub, ast.Name) and sub.id == "context" ), (ref, ast.unparse(item)) - # The phase the run evaluates is the one it checked. + assert not exits(item), (ref, ast.unparse(item)) + # The phase the run evaluates is the one it checked: the run binds it + # once, before the check, anywhere in its body, nested scopes included. + bound = [ + ast.unparse(item) + for item in run.body + for sub in ast.walk(item) + if binds_phase(sub) + ] + assert bound == ["phase = str(context.params['phase'])"], (ref, bound) evaluated = [ sub for item in run.body[index + 1 :] @@ -2362,14 +2405,49 @@ def test_each_gate_kernel_checks_its_phase_code_first() -> None: ), ref +def test_the_phase_code_check_refuses_what_its_tables_leave_out() -> None: + """The stale-code check's two other branches (microcosm#931, ruled 9 Nov + (late)). A phase its battery's table leaves out computes through nothing + of its own, so its node passes carrying no ``_code`` and is + refused carrying one, whatever its value; a battery with no table is + refused. Both tables cover every phase their kernels run today, so no + gate node reaches either branch.""" + from microcosm.build.uk_runtime.gate_phase_codes import ( + UK_FULL_GATE_PHASE_CODE, + UK_SPINE_GATE_PHASE_CODE, + require_current_phase_code, + ) + + tables = {"spine": UK_SPINE_GATE_PHASE_CODE, "full": UK_FULL_GATE_PHASE_CODE} + for battery, other in (("spine", "full"), ("full", "spine")): + for phase, code in tables[other].items(): + assert phase not in tables[battery], (battery, phase) + require_current_phase_code({"phase": phase}, phase, battery=battery) + for value in ("0" * 64, code()): + with pytest.raises( + ValueError, match=f"UK {battery} gate {phase}-only code" + ): + require_current_phase_code( + {"phase": phase, f"{phase}_code": value}, + phase, + battery=battery, + ) + for phase, code in UK_SPINE_GATE_PHASE_CODE.items(): + with pytest.raises(ValueError, match="No UK gate battery 'local'"): + require_current_phase_code( + {"phase": phase, f"{phase}_code": code()}, phase, battery="local" + ) + + def test_no_uk_graph_kernel_hashes_the_phase_codes() -> None: """The phase codes and their stale-code check live in gate_phase_codes, which no UK graph kernel's key covers, the gate kernels' implementation - hashes among them (microcosm#931 ruling 3a). A phase's own code reaches - its key through the ``_code`` parameter alone, so an edit to what - one phase alone computes through moves that parameter and re-keys no node - whose kernel hashes graph_evidence or graph_terminal, which define the - gate kernels, and neither of those carries the tables, the code + hashes among them (microcosm#931, ruled 9 Nov (late)). A phase's own code + reaches its key through the ``_code`` parameter alone, so an edit + to a phase's list in gate_phase_codes moves that parameter, which re-keys + that phase's node and every node after it, and moves no kernel's hash, + graph_evidence's and graph_terminal's hashers included: neither of those + modules, which define the gate kernels, carries the tables, the code functions or the check.""" from microcosm.build.uk_runtime import ( gate_phase_codes, @@ -2405,8 +2483,8 @@ def test_no_uk_graph_kernel_hashes_the_phase_codes() -> None: def test_the_household_type_check_refuses_before_the_targets_compile() -> None: - """The household-type check's refusal-only reason (microcosm#931 ruling - 4): ledger_targets' check is listed, its one caller is + """The household-type check's refusal-only reason (microcosm#931, ruled + 9 Nov (late)): ledger_targets' check is listed, its one caller is compile_uk_target_registry, which calls it in a statement of its own, on the contract's targets, before the loop that compiles the references, and the full build's target compilation and the national targets both reach @@ -2485,8 +2563,10 @@ def calls_check(node) -> bool: def test_every_reader_of_the_annuity_and_trust_column_hashes_its_module() -> None: """The annuity and trust column's name lives in annuity_trust_column, - which holds that name alone, and every module that reads it imports it - from there (microcosm#931 ruling 4). The measure resolver reads zeros + which holds that name alone, and every uk_runtime module that reads it + imports it from there, or reads it as an attribute of + annuity_trust_column itself, never of another module that imports it + (microcosm#931, ruled 9 Nov (late)). The measure resolver reads zeros where a frame lacks the column, so a renamed constant would change the measures without re-keying them, and the name cannot be listed as unbound. Every stage and kernel whose code reads it hashes the module: @@ -2495,7 +2575,13 @@ def test_every_reader_of_the_annuity_and_trust_column_hashes_its_module() -> Non and the dense measures and the national problem, which compute through nothing in frs_spine, so their keys leave it out (test_uk_target_compile_modules holds both kernels' hashes to an edit to - either module).""" + either module). frs_spine itself is held as the household-type test holds + frs_relationships: every key that covers it is the create kernel's or a + spine stage's, and neither runs on the local or the national line on a + bound spine, whose root is the bound spine checkpoint, so an edit to + frs_spine re-keys no node of either line there.""" + import ast + from microcosm.build.uk_runtime import annuity_trust_column, frs_spine column = (annuity_trust_column.__name__, "ANNUITY_AND_TRUST_INCOME_COLUMN") @@ -2512,6 +2598,17 @@ def test_every_reader_of_the_annuity_and_trust_column_hashes_its_module() -> Non facts = _module_definitions(name, _HELPER_SCOPE) assert column[1] not in facts.definitions, name assert facts.imports.get(column[1], column) == column, name + through = sorted( + ast.unparse(node) + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))) + if isinstance(node, ast.Attribute) + and node.attr == column[1] + and not ( + isinstance(node.value, ast.Name) + and facts.modules.get(node.value.id) == column[0] + ) + ) + assert not through, (name, through) stages = { stage: annuity_trust_column.__name__ in hashed for stage, (_, hashed, walk) in _stage_walks().items() @@ -2533,6 +2630,16 @@ def test_every_reader_of_the_annuity_and_trust_column_hashes_its_module() -> Non assert frs_spine.__name__ not in walk.reach, label assert not [item for item in walk.stops if item[0] == frs_spine.__name__] assert frs_spine.__name__ not in key, label + keyed = sorted( + label + for walks in _kernel_rule_walks().values() + for label, (_, key) in walks.items() + if frs_spine.__name__ in key + ) + assert "uk.create@1" in keyed, keyed + assert all( + label == "uk.create@1" or label.startswith("uk.stage.") for label in keyed + ), keyed def test_each_gate_battery_walks_the_engine_its_build_hands_it() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py index 658ce82b72..c97e9e133f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py @@ -620,10 +620,13 @@ def _spine_gate_context(phase: str, **changes): def test_each_spine_gate_phase_refuses_a_stale_or_missing_phase_code(phase, code): """Each spine gate phase's node carries its own code, which the kernel checks before it reads anything of its node but its parameters, as the - full gates check theirs (microcosm#931 ruling 3b): a graph declaration - replayed from another process cannot key a spine gate on stale phase - code. The node's own declared code passes the check, and the kernel goes - on to read its artifacts.""" + full gates check theirs (microcosm#931, ruled 9 Nov (late)): a spine gate + node whose ``_code`` is stale or missing is refused whenever it + runs, and no spine gate evaluates its phase under a key that names other + phase code. A record stored under the node's key is reused without + running the kernel, so the check does not guard a cache hit. The node's + own declared code passes the check, and the kernel goes on to read its + artifacts.""" kernel, declared = _spine_gate_context(phase) with pytest.raises(_ArtifactReadError): kernel.run(declared) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index 1733c07896..a9aa3dfd12 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -1569,7 +1569,7 @@ def test_each_gate_phase_refuses_a_stale_or_missing_phase_code(phase, code) -> N """Each full gate phase's node carries its own code, which the kernel checks before it evaluates anything (microcosm#931 ruling 6), through the check the spine gates share (gate_phase_codes.require_current_phase_code; - ruling 3).""" + microcosm#931, ruled 9 Nov (late)).""" from types import SimpleNamespace from microcosm.build.uk_runtime.graph_terminal import ( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index b66bdb249a..f677183748 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -2234,22 +2234,28 @@ def test_the_hold_counts_every_weight_change_after_the_split(self) -> None: # spi_support's two helpers, a wrapper or an alias: # - a wrapper is a function of any of the three stages' modules named # _exact... that refers to one; - # - an alias is a name one of those modules binds to one outside every - # function, at module level or in a class body: imported under - # another name (from ... import _importance_weights_with_exact_total - # as _total), or assigned from an expression that refers to one + # - an alias is a name one of those modules, or spi_support, which + # defines the helpers, binds to one outside every function, at + # module level or in a class body: imported under another name + # (from ... import _importance_weights_with_exact_total as _total), + # or assigned from an expression that refers to one # (_total = _importance_weights_with_exact_total, or a # functools.partial of it). - # Wrappers and aliases are found to a fixed point over the three + # Wrappers and aliases are found to a fixed point over those four # modules, so one module's wrapper or alias counts where another module # refers to it, and a reference to an alias counts as a reference to a - # wrapper does. Every function, method and nested function counts its - # own references, not those of the functions it nests. A lambda's + # wrapper does. Only the three stages' modules' functions are counted. + # Every function, method and nested function counts its own + # references, not those of the functions it nests. A lambda's # reference counts in the function that writes it, and a local alias, a # local partial or a local import under another name counts once, where # it names the helper or wrapper. What is counted is references, not # calls: a loop that corrects twice, or a local alias called twice, - # counts once. + # counts once. Wrappers are looked for only in the three stages' + # modules, so a correction made through a function of another module + # that corrects a total itself (spi_band_donors' _exact_incumbent_total, + # or the spi_support functions that call the fallback) is not counted, + # nor one through a helper looked up by a name built at run time. helpers = {"_exact_total_correction", "_importance_weights_with_exact_total"} stages = ("cgt_support_split", *weighing) modules = {graph_kernels._stage_module(stage) for stage in stages} @@ -2344,8 +2350,12 @@ def aliases(tree, counted: set[str]) -> set[str]: } == {function.name for function in functions.values()} every[module.__name__] = functions trees[module.__name__] = tree + # spi_support defines the helpers and can alias them too, so its + # bindings outside every function join the alias scan; its functions + # are not among the counted correction sites. + trees[spi_support.__name__] = ast.parse(inspect.getsource(spi_support)) # The names a reference to which counts: the helpers, the wrappers' - # names and the aliases, to a fixed point over all three modules. + # names and the aliases, to a fixed point over the four modules. counted = set(helpers) while True: wrappers = { diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py index 2fb115ffaf..9a59d4fc25 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_target_compile_modules.py @@ -323,9 +323,9 @@ def test_an_edit_to_frs_spine_or_frs_relationships_leaves_the_kernels_keys( tmp_path, monkeypatch, kernel, module, edited_module ): """No target or problem kernel hashes frs_spine or frs_relationships - (microcosm#931 ruling 4). The dense measures and the national problem - read nothing of frs_spine's but the annuity and trust column's name, - which lives in annuity_trust_column. The target compilation and the + (microcosm#931, ruled 9 Nov (late)). The dense measures and the national + problem read nothing of frs_spine's but the annuity and trust column's + name, which lives in annuity_trust_column. The target compilation and the national targets reach the ONS household type domain only through ledger_targets' household-type check, which only refuses and is listed refusal-only in test_uk_graph. So on a bound spine an edit to either file @@ -348,7 +348,7 @@ def test_an_edit_to_the_annuity_and_trust_column_rekeys_the_measures( """The measure resolver reads zeros where a frame lacks the annuity and trust column, so a renamed constant would change the measures without re-keying them: the dense measures and the national problem hash the - column's module (microcosm#931 ruling 4).""" + column's module (microcosm#931, ruled 9 Nov (late)).""" from microcosm.build.uk_runtime import annuity_trust_column kernel, _ = _KERNELS_BY_REF[ref] From a3a978e5d44ae76bddbbb3dad396553d1d0369c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:08:06 +0000 Subject: [PATCH 053/108] Check every import of the annuity column's name, and say where the premises test looks (#931) The re-review of e5d0accca left two nits, both in tests. This commit takes them. It moves nothing: no kernel hash, node key, value, declaration or generated file. - test_every_reader_of_the_annuity_and_trust_column_hashes_its_module looked an import of the name up by the name it binds, since facts.imports is keyed that way. So an aliased import through frs_spine or spi_spine, both of which import the name, passed it, although its docstring and the changelog's column bullet say every module reads the name from annuity_trust_column. The test now checks every from-import of the name, whatever name it binds, resolving a relative import as module_definitions does, which also catches two imports that bind one name. Keys were safe either way: the kernel rule follows frs_spine's import of the name to annuity_trust_column, and at the tip all four readers import it from there. - The premises test's comment said wrappers and aliases are found to a fixed point over four modules, and then that wrappers are looked for only in the three stages' modules, which is what the code does. It now says aliases are found over the four modules and wrappers over the three, together to a fixed point, and so does the comment over the counted names. Tests (engine-free; code facts only): test_uk_graph and test_uk_identity_stability_receipts pass (154 tests), and ruff is clean on both. On a scratch copy of e5d0accca with this commit laid over it, the re-review's three mutations now fail the reader test alone: the name imported under another name through frs_spine, or through spi_spine, in place of measure_simulation's own import, and the same import through frs_spine added beside it. The first two ran with the kernel, stage and name rules and both target-compile edit tests, the third with the whole of test_uk_graph and test_uk_target_compile_modules. The unmutated copy passes. Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/uk/test_uk_graph.py | 27 ++++++++++++++++--- .../uk/test_uk_identity_stability_receipts.py | 12 +++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index db41b23592..47c171c3cb 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -2564,8 +2564,9 @@ def calls_check(node) -> bool: def test_every_reader_of_the_annuity_and_trust_column_hashes_its_module() -> None: """The annuity and trust column's name lives in annuity_trust_column, which holds that name alone, and every uk_runtime module that reads it - imports it from there, or reads it as an attribute of - annuity_trust_column itself, never of another module that imports it + imports it from there, under whatever name the import binds, or reads it + as an attribute of annuity_trust_column itself, never of another module + that imports it (microcosm#931, ruled 9 Nov (late)). The measure resolver reads zeros where a frame lacks the column, so a renamed constant would change the measures without re-keying them, and the name cannot be listed as @@ -2588,6 +2589,15 @@ def test_every_reader_of_the_annuity_and_trust_column_hashes_its_module() -> Non holder = _module_definitions(column[0], _HELPER_SCOPE).definitions assert set(holder) == {column[1]} assert _helper_kind(*column) == "name" + + def origin(node: ast.ImportFrom) -> str: + # Resolved as _module_definitions resolves it; every uk_runtime + # module's package is frs_spine's. + if not node.level: + return node.module or "" + base = frs_spine.__package__.rsplit(".", node.level - 1)[0] + return ".".join(part for part in (base, node.module) if part) + runtime = Path(frs_spine.__file__).parent for path in sorted(runtime.glob("*.py")): if path.stem == "annuity_trust_column": @@ -2598,9 +2608,20 @@ def test_every_reader_of_the_annuity_and_trust_column_hashes_its_module() -> Non facts = _module_definitions(name, _HELPER_SCOPE) assert column[1] not in facts.definitions, name assert facts.imports.get(column[1], column) == column, name + tree = ast.parse(path.read_text(encoding="utf-8")) + # Every import of the name, whatever name it binds: facts.imports is + # keyed by the bound name, so it misses an aliased import through + # frs_spine, and keeps only the last of two imports that bind one name. + origins = { + origin(node) + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + and any(alias.name == column[1] for alias in node.names) + } + assert origins <= {column[0]}, (name, sorted(origins)) through = sorted( ast.unparse(node) - for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))) + for node in ast.walk(tree) if isinstance(node, ast.Attribute) and node.attr == column[1] and not ( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index f677183748..b0c417f20e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -2241,10 +2241,11 @@ def test_the_hold_counts_every_weight_change_after_the_split(self) -> None: # or assigned from an expression that refers to one # (_total = _importance_weights_with_exact_total, or a # functools.partial of it). - # Wrappers and aliases are found to a fixed point over those four - # modules, so one module's wrapper or alias counts where another module - # refers to it, and a reference to an alias counts as a reference to a - # wrapper does. Only the three stages' modules' functions are counted. + # Aliases are found over those four modules and wrappers over the three + # stages' modules, together to a fixed point, so one module's wrapper + # or alias counts where another module refers to it, and a reference to + # an alias counts as a reference to a wrapper does. Only the three + # stages' modules' functions are counted. # Every function, method and nested function counts its own # references, not those of the functions it nests. A lambda's # reference counts in the function that writes it, and a local alias, a @@ -2355,7 +2356,8 @@ def aliases(tree, counted: set[str]) -> set[str]: # are not among the counted correction sites. trees[spi_support.__name__] = ast.parse(inspect.getsource(spi_support)) # The names a reference to which counts: the helpers, the wrappers' - # names and the aliases, to a fixed point over the four modules. + # names and the aliases, to a fixed point: aliases over the four + # modules, wrappers over the three stages' modules. counted = set(helpers) while True: wrappers = { From 701ae514e096346145e98949a033137fd49d6f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:08:06 +0000 Subject: [PATCH 054/108] Say the Lifetime ISA rank blends no income rank, and that tied balances share a rank (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 6 of docs/uk-isa-subscriptions-1415.md has said since 960cd0ab9 that the Lifetime ISA payment rank blends the balance and income ranks with a keyed uniform. The code never did: since the stage was added (bf0abdc37), draw_lifetime_isa_payments has blended only the weighted rank of the person's Lifetime ISA balance, over every holder aged 18 to 47 rather than within income bands, with a keyed uniform at the same balance_rank_weight as the subscription amounts. Step 6 now says so (María's ruling 4 of 10 Nov). Step 3 also says, in the module docstring's words, that holders tied on a balance share its rank, so the keyed uniform alone orders them: since 0c850b1f3, _weighted_rank gives a tie group one mid-rank, and both draws blend it with a keyed uniform. Ruling 1 of 10 Nov has stage 53's declaration state ties in a later change. Text only: no code, declaration, generated file or key moves. Co-Authored-By: Claude Opus 5.5 --- docs/uk-isa-subscriptions-1415.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/uk-isa-subscriptions-1415.md b/docs/uk-isa-subscriptions-1415.md index 645cd99458..879ced1299 100644 --- a/docs/uk-isa-subscriptions-1415.md +++ b/docs/uk-isa-subscriptions-1415.md @@ -82,7 +82,8 @@ How the tables relate, on HMRC's own figures: (`shortfall_weight_unmet`); no holder is invented. 3. **How much.** Within each income band and class, amounts follow Table 9.7's subscription bands: uniform inside a band, exactly £20,000 at the top. The - rank blends the person's balance rank with a keyed uniform. + rank blends the person's balance rank with a keyed uniform. Holders tied on a + balance share its rank, so the keyed uniform alone orders them. 4. **The both class.** Each person's amount is split by their balance shares, shifted on the logit scale so the class's stocks-and-shares total equals Table 9.4's 2023-24 stocks-and-shares amount less the stocks-and-shares-only @@ -102,7 +103,8 @@ How the tables relate, on HMRC's own figures: 2017. Amounts lie in (0, £4,000], drawn by a truncated-exponential tilt so the total equals Table 9.4's 2024-25 Lifetime ISA amount (£2,818m from 1,136,000 accounts), so the baseline leaves nothing above the overall limit. - The rank blends the balance and income ranks with a keyed uniform. + The rank blends the person's Lifetime ISA balance rank with a keyed uniform, + as in step 3, but over every holder aged 18 to 47, not by income band. ## Decisions (28 October 2026) From 76909e5349ed1f8b8300261460ca66e041c1bb38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:16:22 +0000 Subject: [PATCH 055/108] Widen the Child Benefit gate's per-age tolerance to three standard errors at thin ages (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UK Child Benefit take-up stage gate (uk_stage_child_benefit_take_up) held the claimed share of eligible children at each measured single year of age within a fixed maximum_age_claim_rate_deviation, 0.05, of HMRC's published rate for it. A measured age is one with at least minimum_age_child_rows child rows whose solved rate was not clipped. Each measured age is now held within the larger of that deviation and z standard errors of a share drawn at the age's published rate p over its n child rows, z * sqrt(p (1 - p) / n) (María's ruling of 10 Nov). p and n are the age's published_rate and eligible_child_rows in the receipt the gate already reads. z is the new declared parameter maximum_age_claim_rate_standard_errors, 3 in gates.json. The stage-health binding now routes it, since the battery refuses a declared parameter its binding does not route. Three standard errors exceed 0.05 only below 3,600 p (1 - p) child rows, so an age with more rows keeps the fixed deviation and only thin ages loosen. The band treats an age's child rows as independent draws at p, an approximation as ruled: in the stage, siblings share their family's draw, reporters always claim and the realised rate is weighted. The overall claim-rate check and the opt-out check do not change. Why: group (a)'s keyed re-draws move each age's realised claim rate by sampling noise. At an age with few child rows that noise can exceed the fixed deviation, and on a full K = 1 spine at (a)'s head it did, at one thin age, while the pre-(a) head and a copy-ids-only head passed. A private read of the stage evidence found draw noise, not a shift. The draws are keyed, so the failure is deterministic and would repeat on every rebuild, including the national build. The gate now allows for each age's own sampling noise. What moves: the gate's verdicts at thin ages. An age whose realised rate is more than the fixed deviation from its published rate, but within its three standard errors, now passes. A failing age's line names the tolerance it was held to, "the larger of 0.05 and 3 standard errors of a share at the published rate over that age's child rows", and the gate's details count the measured ages held above the fixed deviation (ages_tolerance_above_floor). At a published rate of 0 or 1 the binomial term is 0, so the floor holds. No stage output or generated file moves: the gate reads the receipt and draws nothing. - Of the 99 UK kernel hashes 14 move, and no phase code: the spine gates' (stage_health, battery_bindings and the solve fingerprint, which gates.json is part of); uc_deduction_attributes' and uc_take_up's (stage_health); the full build's target compilation, selection, measures, problem and rotated holdout (the solve fingerprint), its preflight and calibrated gates (battery_bindings and the solve or country fingerprint) and its certification readiness (microcosm.data.contract, which the re-pin edits); and the national targets, problem and gates (the country fingerprint; the gates also hash the gate registry's modules). - Node keys move from the assembled spine gate on: 64 of the gated spine's 81 nodes. On a bound spine the full build re-keys from its target compilation on, so it re-solves, and the national line re-keys all but the bound checkpoint. - The contract re-pins the gate battery's policy, gates-manifest and spec-fingerprint digests and the release certification's spine part, and test_contract mirrors the battery pins. The calibration seam, release cut and dense line digests do not move. Tests (engine-free; invented values only): a thin age passes inside its three standard errors and fails outside them with the new line; a large age keeps the fixed deviation; a published rate of 0 or 1 keeps it even at the fewest rows the gate measures; the fixture's ages carry a weighted child mass far from their row counts, so a gate that took that mass for n would fail on value; the gate refuses a missing multiple with a KeyError and a null, zero, negative, NaN or infinite one with a ValueError; and the committed declaration's parameters equal the test's. test_uk_stage_health, the data shard's test_contract, test_gate_battery_contract_pins, test_uk_battery_bindings and test_uk_release_certification pass on the final tree (565 tests). ruff check and ruff format --check are clean on the five touched Python files. Co-Authored-By: Claude Opus 5.5 --- ...ild-benefit-age-noise-tolerance.changed.md | 6 + .../src/microcosm/build/uk/gates.json | 3 +- .../build/uk_runtime/battery_bindings.py | 5 +- .../build/uk_runtime/stage_health.py | 47 ++++- .../engine_free/uk/test_uk_stage_health.py | 165 +++++++++++++----- .../src/microcosm/data/contract.py | 10 +- .../tests/engine_free/shared/test_contract.py | 6 +- 7 files changed, 186 insertions(+), 56 deletions(-) create mode 100644 changelog.d/931-uk-child-benefit-age-noise-tolerance.changed.md diff --git a/changelog.d/931-uk-child-benefit-age-noise-tolerance.changed.md b/changelog.d/931-uk-child-benefit-age-noise-tolerance.changed.md new file mode 100644 index 0000000000..3e70221ee3 --- /dev/null +++ b/changelog.d/931-uk-child-benefit-age-noise-tolerance.changed.md @@ -0,0 +1,6 @@ +The UK Child Benefit take-up stage gate (`uk_stage_child_benefit_take_up`) now holds each measured age within the larger of 0.05 and three standard errors of a share drawn at its published rate over its child rows (#931; María's ruling of 10 Nov). A measured age, one with at least `minimum_age_child_rows` child rows whose solved rate was not clipped, was held within a fixed `maximum_age_claim_rate_deviation`, 0.05, of HMRC's published rate for it. The band is `maximum_age_claim_rate_standard_errors` × sqrt(p (1 − p) / n), with the new parameter declared as 3, and p and n the receipt's `published_rate` and `eligible_child_rows` for the age. It treats the age's child rows as independent draws at p, an approximation as ruled: in the stage, siblings share their family's draw, reporters always claim and the realised rate is weighted. + +- **Why.** #931's keyed redraws move each age's realised claim rate by sampling noise. At an age with few child rows that noise can exceed a fixed 0.05, and because the draws are keyed, such a failure would repeat on every rebuild, including the national build. +- **Floor.** Three standard errors exceed 0.05 only below 3,600 p (1 − p) child rows, so an age with more rows keeps 0.05 and only thin ages loosen. At a published rate of 0 or 1 the term is 0, so the floor holds. The overall claim-rate check and the opt-out check are unchanged. +- **Report.** A failing age's line names the tolerance used at that age, "(tolerance t: the larger of 0.05 and 3 standard errors of a share at the published rate over that age's child rows)", and the gate's details count the measured ages held to more than the floor (`ages_tolerance_above_floor`). The gate refuses a missing, non-finite or non-positive `maximum_age_claim_rate_standard_errors`. The stage-health binding now routes the parameter, since the battery refuses a declared parameter its binding does not route. +- **What moves.** The gate's verdicts at thin ages: an age whose realised rate is more than 0.05 from its published rate but within its three standard errors now passes. No stage output or generated file moves: the gate reads the receipt and draws nothing. Of the 99 UK kernel hashes 14 move, and no phase code: the spine gates' (`stage_health`, `battery_bindings` and the solve fingerprint, which `gates.json` is part of); `uc_deduction_attributes`' and `uc_take_up`'s (`stage_health`); the full build's target compilation, selection, measures, problem and rotated holdout (the solve fingerprint), its preflight and calibrated gates (`battery_bindings` and the solve or country fingerprint) and its certification readiness (`microcosm.data.contract`, which the re-pin edits); and the national targets, problem and gates (the country fingerprint; the gates also hash the gate registry's modules). Node keys move from the assembled spine gate on: 64 of the gated spine's 81 nodes. On a bound spine the full build re-keys from its target compilation on, so it re-solves, and the national line re-keys all but the bound checkpoint. The contract re-pins the gate battery's policy, gates-manifest and spec-fingerprint digests and the release certification's spine part; the calibration seam, release cut and dense line digests do not move. diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 723e4de542..a45ca42963 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -684,11 +684,12 @@ "check": "child_benefit_take_up", "maximum_claim_rate_deviation": 0.02, "maximum_age_claim_rate_deviation": 0.05, + "maximum_age_claim_rate_standard_errors": 3, "minimum_age_child_rows": 200, "maximum_opt_out_share_deviation": 0.01, "minimum_eligible_family_units": 1 }, - "notes": "Stage-time gate over the Child Benefit take-up receipt (microcosm#1063): the share of eligible children claimed for is within 0.02 of HMRC's published claim rates by single year of age (Table 15, May 2025) at the frame's age mix, and within 0.05 at each age with at least 200 child rows whose solved rate was not clipped (clipped ages are reported); the opted-out share of claiming families is within 0.01 of the published opted-out share of registered families (August 2025), or below it only where the charged families are exhausted. The terminal take-up signal gate no longer measures would_claim_child_benefit or child_benefit_opts_out, whose rates are shares of eligible children and of claiming families, not of every benefit unit." + "notes": "Stage-time gate over the Child Benefit take-up receipt (microcosm#1063): the share of eligible children claimed for is within 0.02 of HMRC's published claim rates by single year of age (Table 15, May 2025) at the frame's age mix. At each age with at least 200 child rows whose solved rate was not clipped (clipped ages are reported), the share is within a tolerance of that age's published rate p: the larger of 0.05 and 3 standard errors of a share drawn at p over the age's n child rows, 3 * sqrt(p * (1 - p) / n) (maximum_age_claim_rate_standard_errors; María's ruling of 2026-11-10, microcosm#931). Each age's realised rate carries the sampling noise of the stage's keyed draws, which at an age with few child rows can exceed a fixed 0.05; because the draws are keyed, a failure from that noise would repeat on every rebuild. Three standard errors exceed 0.05 only below 3,600 * p * (1 - p) child rows, so an age with more rows keeps 0.05. The opted-out share of claiming families is within 0.01 of the published opted-out share of registered families (August 2025), or below it only where the charged families are exhausted. The terminal take-up signal gate no longer measures would_claim_child_benefit or child_benefit_opts_out, whose rates are shares of eligible children and of claiming families, not of every benefit unit." }, { "id": "uk_stage_cgt_incidence_clone_mass", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 25cd1b25a0..1b37ad7bc6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -2135,9 +2135,12 @@ def _ledger_compile_parity_required_artifacts( "zeroed_columns", "restored_columns", "restored_within_earnings_limit_columns", - # microcosm#1063 child_benefit_take_up stage-health check. + # microcosm#1063 child_benefit_take_up stage-health check; its + # per-age standard errors from María's ruling of 2026-11-10 + # (microcosm#931). "maximum_claim_rate_deviation", "maximum_age_claim_rate_deviation", + "maximum_age_claim_rate_standard_errors", "minimum_age_child_rows", "maximum_opt_out_share_deviation", "minimum_eligible_family_units", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index 2fbbd61472..f89636b609 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -1058,9 +1058,19 @@ def _child_benefit_take_up_gate( published rates at the frame's own age mix, and at each single year of age with enough child rows to measure it; an age whose solved rate was clipped (reporters alone exceed the published rate, or every family claiming - falls short of it) is reported, not held. The opted-out share of claiming - families is held to the published share, or may fall short of it only - where the receipt shows the charged families exhausted (microcosm#1063). + falls short of it) is reported, not held. A measured age is held to the + larger of ``maximum_age_claim_rate_deviation`` and + ``maximum_age_claim_rate_standard_errors`` standard errors of a share + drawn at its published rate p over its n child rows, + ``sqrt(p (1 - p) / n)``, since the stage's keyed draws move each age's + realised rate by sampling noise that at an age with few child rows can + exceed the fixed deviation; an age with many rows keeps the fixed + deviation (María's ruling of 2026-11-10, microcosm#931). The band treats + the age's child rows as independent draws at p, an approximation as + ruled: siblings share their family's draw, reporters always claim and the + realised rate is weighted. The opted-out share of claiming families is + held to the published share, or may fall short of it only where the + receipt shows the charged families exhausted (microcosm#1063). """ check = "child_benefit_take_up" @@ -1072,6 +1082,15 @@ def _child_benefit_take_up_gate( parameters["maximum_age_claim_rate_deviation"], label=f"{stage}.maximum_age_claim_rate_deviation", ) + age_standard_errors = _finite_number( + parameters["maximum_age_claim_rate_standard_errors"], + label=f"{stage}.maximum_age_claim_rate_standard_errors", + ) + if age_standard_errors <= 0.0: + raise ValueError( + f"{stage}.maximum_age_claim_rate_standard_errors must be positive, " + f"got {age_standard_errors:g}." + ) minimum_age_rows = int(parameters["minimum_age_child_rows"]) opt_out_tolerance = _finite_number( parameters["maximum_opt_out_share_deviation"], @@ -1104,13 +1123,15 @@ def _child_benefit_take_up_gate( worst: tuple[float, int] | None = None clipped: list[int] = [] measured = 0 + above_floor = 0 for row in ages: row = _mapping(row, label=f"{stage}.claims.ages[]") age = int(row.get("age", -1)) if row.get("clipped") is True: clipped.append(age) continue - if int(row.get("eligible_child_rows", 0)) < minimum_age_rows: + rows = int(row.get("eligible_child_rows", 0)) + if rows < minimum_age_rows: continue published = _finite_number( row.get("published_rate"), label=f"{stage}.age {age}.published_rate" @@ -1119,16 +1140,30 @@ def _child_benefit_take_up_gate( row.get("realized_rate"), label=f"{stage}.age {age}.realized_rate" ) measured += 1 + # The age's own sampling noise where it exceeds the floor. At a + # published rate of 0 or 1 the binomial term is 0, and a rate outside + # [0, 1] or an age without rows has none, so the floor holds. + tolerance = age_tolerance + if rows > 0 and 0.0 <= published <= 1.0: + tolerance = max( + age_tolerance, + age_standard_errors * math.sqrt(published * (1.0 - published) / rows), + ) + if tolerance > age_tolerance: + above_floor += 1 deviation = abs(achieved - published) if worst is None or deviation > worst[0]: worst = (deviation, age) - if deviation > age_tolerance: + if deviation > tolerance: failures.append( f"{stage}: at age {age} {achieved:.4f} of eligible children are " f"claimed for against the published {published:.4f} (tolerance " - f"{age_tolerance})." + f"{tolerance:.4f}: the larger of {age_tolerance} and " + f"{age_standard_errors:g} standard errors of a share at the " + "published rate over that age's child rows)." ) details["ages_measured"] = measured + details["ages_tolerance_above_floor"] = above_floor details["clipped_ages"] = clipped if worst is not None: details["largest_age_deviation"] = {"age": worst[1], "deviation": worst[0]} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py index 3855402b05..b3070d785f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py @@ -2703,50 +2703,69 @@ def gate(payload): assert not gate(payload).passed -def test_child_benefit_take_up_holds_claims_by_age_and_the_opt_out_share() -> None: - parameters = { - "stage": "child_benefit_take_up", - "check": "child_benefit_take_up", - "maximum_claim_rate_deviation": 0.02, - "maximum_age_claim_rate_deviation": 0.05, - "minimum_age_child_rows": 200, - "maximum_opt_out_share_deviation": 0.01, - "minimum_eligible_family_units": 1, - } +_CHILD_BENEFIT_PARAMETERS = { + "stage": "child_benefit_take_up", + "check": "child_benefit_take_up", + "maximum_claim_rate_deviation": 0.02, + "maximum_age_claim_rate_deviation": 0.05, + "maximum_age_claim_rate_standard_errors": 3, + "minimum_age_child_rows": 200, + "maximum_opt_out_share_deviation": 0.01, + "minimum_eligible_family_units": 1, +} - def age(value, published, realized, *, rows=1_000, clipped=False): - return { - "age": value, - "published_rate": published, + +def _child_benefit_age(value, published, realized, *, rows=1_000, clipped=False): + return { + "age": value, + "published_rate": published, + "realized_rate": realized, + "eligible_child_rows": rows, + # A weighted child mass far from the row count, so a gate that took it + # for n would fail the sampling-noise test on value. + "eligible_children": rows * 250.0, + "clipped": clipped, + } + + +def _child_benefit_gate( + *, + realized=0.868, + ages=None, + share=0.0905, + exhausted=False, + units=100, + parameters=_CHILD_BENEFIT_PARAMETERS, +): + evidence = { + "stage": "child_benefit_take_up", + "claims": { + "eligible_family_units": units, + "target_rate": 0.871, "realized_rate": realized, - "eligible_child_rows": rows, - "clipped": clipped, - } + "ages": ages + if ages is not None + else [ + _child_benefit_age(0, 0.688, 0.69), + _child_benefit_age(1, 0.768, 0.75), + ], + }, + "opt_outs": { + "target_share": 0.0907, + "realized_share": share, + "pool_exhausted": exhausted, + }, + } + return uk_stage_health_gate( + evidence=evidence, + stage="child_benefit_take_up", + check="child_benefit_take_up", + parameters=parameters, + ) - def gate(*, realized=0.868, ages=None, share=0.0905, exhausted=False, units=100): - evidence = { - "stage": "child_benefit_take_up", - "claims": { - "eligible_family_units": units, - "target_rate": 0.871, - "realized_rate": realized, - "ages": ages - if ages is not None - else [age(0, 0.688, 0.69), age(1, 0.768, 0.75)], - }, - "opt_outs": { - "target_share": 0.0907, - "realized_share": share, - "pool_exhausted": exhausted, - }, - } - return uk_stage_health_gate( - evidence=evidence, - stage="child_benefit_take_up", - check="child_benefit_take_up", - parameters=parameters, - ) +def test_child_benefit_take_up_holds_claims_by_age_and_the_opt_out_share() -> None: + age, gate = _child_benefit_age, _child_benefit_gate result = gate() assert _passed(result) assert result.details["ages_measured"] == 2 @@ -2768,6 +2787,72 @@ def gate(*, realized=0.868, ages=None, share=0.0905, exhausted=False, units=100) assert not gate(ages=[]).passed +def test_child_benefit_take_up_holds_each_age_to_its_own_sampling_noise() -> None: + """María's ruling of 2026-11-10 (microcosm#931): a measured age is held to + the larger of the 0.05 floor and three standard errors of a share drawn at + its published rate over its child rows, so only thin ages loosen. The age, + rates and row counts here are invented.""" + + assert _gate_parameters("uk_stage_child_benefit_take_up") == ( + _CHILD_BENEFIT_PARAMETERS + ) + + def one_age(published, realized, rows): + return _child_benefit_gate( + ages=[_child_benefit_age(3, published, realized, rows=rows)] + ) + + # At 0.6 over 240 rows p (1 - p) / n is 0.001, so three standard errors + # are about 0.0949: a deviation of 0.06 passes there and 0.10 fails. + thin = one_age(0.6, 0.66, 240) + assert _passed(thin), thin.failures + assert thin.details["ages_tolerance_above_floor"] == 1 + assert one_age(0.6, 0.70, 240).failures == ( + "child_benefit_take_up: at age 3 0.7000 of eligible children are " + "claimed for against the published 0.6000 (tolerance 0.0949: the " + "larger of 0.05 and 3 standard errors of a share at the published rate " + "over that age's child rows).", + ) + # Over 2,400 rows three standard errors are 0.03, below the floor, so the + # age keeps 0.05 and the same deviation of 0.06 fails. + large = one_age(0.6, 0.66, 2_400) + assert large.details["ages_tolerance_above_floor"] == 0 + assert "(tolerance 0.0500: the larger of 0.05 and 3" in " ".join(large.failures) + assert _passed(one_age(0.6, 0.64, 2_400)) + # A published rate of 0 or 1 has no binomial term, so even an age with the + # fewest rows the gate measures keeps the floor. + for published, inside, outside in ((0.0, 0.04, 0.06), (1.0, 0.96, 0.94)): + assert _passed(one_age(published, inside, 200)) + edge = one_age(published, outside, 200) + assert edge.details["ages_tolerance_above_floor"] == 0 + assert "(tolerance 0.0500:" in " ".join(edge.failures) + + +@pytest.mark.parametrize( + "standard_errors", ["absent", None, 0, -3, float("nan"), float("inf")] +) +def test_child_benefit_take_up_refuses_missing_or_non_positive_standard_errors( + standard_errors, +) -> None: + parameters = dict(_CHILD_BENEFIT_PARAMETERS) + if standard_errors == "absent": + # The gate reads it as it reads its other parameters, so a missing key + # raises a KeyError that names it. + del parameters["maximum_age_claim_rate_standard_errors"] + refused = pytest.raises( + KeyError, match="maximum_age_claim_rate_standard_errors" + ) + else: + parameters["maximum_age_claim_rate_standard_errors"] = standard_errors + refused = pytest.raises( + ValueError, + match=r"maximum_age_claim_rate_standard_errors must be (finite|positive)", + ) + + with refused: + _child_benefit_gate(parameters=parameters) + + _PROPERTY_COMPONENTS_PARAMETERS = { "stage": "property_components", "check": "property_components", diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 18a4aa50fd..513b10d705 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -667,13 +667,13 @@ def _is_hierarchy_diagnostics_version(value: object) -> bool: # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "6b58c92b17862758f0d994e7ef72d4fad270c3db9667ad84b92cf552c5fc581e" + "d9c57335c0b9bc5f807fee41b17ed5283d59998962d1cdaec6b1d29cb94c3e61" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "6d54957d276c2738833b27f4cfe67450d430e39267356dc8964c506a25e9904d" + "04416f2a69081a83f5b3e42eb3c3281fc56cea6f65fb302502c6221bd1065b7d" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "e0504f39332bce9be72529e1836a6c5079820ab22a6add5208a0390f589a2ce0" + "edaf8d4dd53bfa1d7558179dceec8e92be5adead65120a93862ce022eaaa1c97" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -1322,10 +1322,10 @@ def ratio(value: object) -> bool: _UK_CERTIFICATION_PART_DIGESTS: Mapping[str, Mapping[str, str]] = { "spine": { "gates_manifest_sha256": ( - "c552c77c2e6b0ca23323996820d9a2f53833fa0377eef4c607bc5fde32e939cf" + "4c3dcf777813db0229de8c545d77ab97925b3eaadc9f6e052fc584eb936d1e25" ), "policy_sha256": ( - "0a2635b7b22034ee1fa9b6574bc259805397171b367f4c7941509648704bf1bb" + "312eaa070ebeadcb09a4fb20090c3fc8698f8b4d9fd52b83e446e1bafa3a3b2c" ), }, "calibration_seam": { diff --git a/packages/microcosm-data/tests/engine_free/shared/test_contract.py b/packages/microcosm-data/tests/engine_free/shared/test_contract.py index dcccc599d7..6d4f3fcb49 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_contract.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_contract.py @@ -212,13 +212,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "6b58c92b17862758f0d994e7ef72d4fad270c3db9667ad84b92cf552c5fc581e" + "d9c57335c0b9bc5f807fee41b17ed5283d59998962d1cdaec6b1d29cb94c3e61" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "6d54957d276c2738833b27f4cfe67450d430e39267356dc8964c506a25e9904d" + "04416f2a69081a83f5b3e42eb3c3281fc56cea6f65fb302502c6221bd1065b7d" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "e0504f39332bce9be72529e1836a6c5079820ab22a6add5208a0390f589a2ce0" + "edaf8d4dd53bfa1d7558179dceec8e92be5adead65120a93862ce022eaaa1c97" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "a6ebc171cadf45b691a2b83632bc1f87f3116ad31aebf073e3786ce90ecc0307" From 289cac93db0e62f91569850b7da341f0253d437b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:09:00 +0000 Subject: [PATCH 056/108] Scope E8's band-donor rerun to the frame stage 17 saw (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E8's band-donor check (_e8_band_donors in tools/verify_uk_identity_stability.py) now scopes the artifact to stage 17 with _frame_as_stage_saw, the helper E5 uses, before it reads the stored layer or reruns the stage (María's go-ahead of 10 Nov). Every row minted after stage 17 is dropped with its people and benefit units: the high-value home copies of stage 26 and the CGT support copies, incidence twins and residential arms of stages 42, 43 and 46. Each one's mass folds back onto the row it came from: arms onto their sources, twins onto their originals by pair sum, then support copies and high-value copies onto their roots. Every household left carries the weight stage 17 left, to rounding, and the scope also reverses the salary-sacrifice and regional-home-value rewrites. The stored seats, carriers and band weights are read on that frame, so a later copy of a donor is no longer read as a donor of its own, and each funding stratum is scaled back from there to the weights the stage read. If the scope leaves in place a household whose id holds an ordinal in a copy-id field above spi_support_channel, the check refuses, by count; the fields come from the copy-id table's own order. e8_identity_receipt now hands the check the artifact instead of cgt_frame, whose salary-sacrifice conversion is already reversed: a second reversal finds every converted record's salary sacrifice at zero and refuses. Why: the check folded only the CGT layers, so the high-value copies stayed in the rerun as households the stage saw, and a donor's copies, which carry its donor flag, were read as donors too. Under the copy-id table's fixed radices each later layer mints in a field of its own above spi_support_channel, so the later copies' ids sort above those of the donors the rerun mints at that field's ordinal 2. Appended after the copies, the new donors broke the Frame's ascending ids, and the rerun (--check e8 with the SPI tape) raised instead of running. Under the old max-id rule the rerun ran, but on the wrong population. What moves: E8's verdict, and with it the band-donor block of the tool's receipt, which now reads each donor once; no other receipt moves. On a spine with high-value copies the rerun completes where it raised before. No package module references the tool and no kernel binds it, so no stage output, node key, kernel hash or generated file moves. The identity tool's changelog fragment gains a bullet for E8, and its What moves bullet now names E8's receipt. The verdict on a real spine is unverified until the next --check e8 run. Tests (engine-free; invented values only), in test_uk_spi_band_donors.py: stage 17's output, with and without geographic clones, goes through the real high-value split and regional home values, then a CGT support split, incidence twins at uneven anchor shares and residential arms replayed through the copy-id table, and a salary-sacrifice conversion. The scope gives back stage 17's rows and weights, the check scales each funding stratum back by the stage's own factor, and the rerun reproduces the stored seats, carriers and band weights with the artifact's people in either order. Mass moved within a donor's family passes, and reweighting the whole family is reported as unequal band weights. A copy that has lost its lineage columns is refused by count. The E8 entry, its other checks stubbed out, hands the band-donor check the artifact. Against the tool at aed673da6 every new test fails, most on the sorted-ids refusal. The touched test file passes; ruff check and ruff format --check are clean. Co-Authored-By: Claude Opus 5.5 --- ...k-identity-tool-high-value-copies.fixed.md | 3 +- .../engine_free/uk/test_uk_spi_band_donors.py | 380 +++++++++++++++++- tools/verify_uk_identity_stability.py | 97 +++-- 3 files changed, 450 insertions(+), 30 deletions(-) diff --git a/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md b/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md index 99cef6fb89..08abc93cf9 100644 --- a/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md +++ b/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md @@ -2,4 +2,5 @@ The UK identity tool (`tools/verify_uk_identity_stability.py`) now scopes its re - **The copies.** The tool's stacking map gains `household_is_high_value_support_copy`, the flag of `high_value_home_support` (stage 26, microcosm#1240). The receipts that scope to an earlier stage now drop the copies and every CGT row made from them: E5's wealth block and E4's FRS-only frame. The copies' mass folds back by family. After the CGT folds, each copy's weight is added onto its root, so the root is back at its weight before the split. No uniform factor could do this, since only the top value stratum is divided. The fold decodes each copy at the copy-id table's `high_value_support` field. It refuses, by count, a copy without a root, a flag that disagrees with the stored index, an index that disagrees with the id, and a family whose decoded size disagrees with `high_value_support_copies`. Dropping the copies while a CGT layer stays in scope is refused. - **E5's stored values.** `regional_home_values` (stage 27, microcosm#1112) rewrites `main_residence_value` and `property_wealth` after the uprating, so E5's fixed-point check of the uprating was comparing stage 27's values. The scoping now reverses that rewrite for every stage before stage 27. The value comes back from `main_residence_value_rank_basis`, which the stage keeps, and the wealth moves back by the same change. An artifact without the rank basis, such as a release, is read as before. The reversal refuses a roster in which a later stage declares either column, since it would then hide that stage's rewrite. -- **What moves.** Tool verdicts only. On the synthetic H2 oracle frame, E5's stored-column check failed with the tool before this change and passes now. On any spine built since `regional_home_values` joined the roster, the check no longer fails on that stage's rewrite. The `was_wealth` scope drops the 115 high-value copies (of 404 rows) it kept. On that frame every high-value root is an SPI row, which E4 drops anyway, so E4 reads the same frame as before, and the E6, E7 and E9 receipts are identical. On a spine whose high-value roots include FRS households, E4's scope and weights change too. No stage, value, node key or generated file moves, since nothing hashes the tool. +- **E8's band donors.** E8 reran the income band donor stage (stage 17) on the frame its CGT checks read, where only the CGT layers were folded away. The high-value copies stayed in the rerun as households the stage saw, and the stored seats, carriers and band weights read a donor's copies as donors of their own. Once the copy-id table gave each layer its own field, the rerun was refused outright: the donors it appended sorted below the copies' ids. The check now scopes the artifact as E5 does (`_frame_as_stage_saw`). Every row minted after stage 17, the high-value copies and every CGT row, is dropped with its people and benefit units, and its mass folds back onto the row it came from. So the check reads the frame stage 17 wrote, at the weights it left, and each donor once, and reruns the stage on that frame less its donors, at the weights the stage read. A row minted after stage 17 that the scope leaves in place is refused by count. +- **What moves.** Tool verdicts only. On the synthetic H2 oracle frame, E5's stored-column check failed with the tool before this change and passes now. On any spine built since `regional_home_values` joined the roster, the check no longer fails on that stage's rewrite. The `was_wealth` scope drops the 115 high-value copies (of 404 rows) it kept. On that frame every high-value root is an SPI row, which E4 drops anyway, so E4 reads the same frame as before, and the E6, E7 and E9 receipts are identical. On a spine whose high-value roots include FRS households, E4's scope and weights change too. E8's band-donor scope moves E8's receipt and no other: on any spine with high-value copies, the check reads each donor once, and its rerun, which needs the SPI tape, now completes where it raised before. No stage, value, node key or generated file moves, since nothing hashes the tool. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index 4916927047..94447cb7a3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util +from types import SimpleNamespace import numpy as np import pandas as pd @@ -10,10 +11,35 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.uk_runtime import spi_band_donors +from microcosm.build.uk_runtime import copy_id_table, spi_band_donors +from microcosm.build.uk_runtime.cgt_residential_split import ( + CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, + HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, +) +from microcosm.build.uk_runtime.cgt_structure import HOUSEHOLD_IS_CGT_CLONE +from microcosm.build.uk_runtime.cgt_support import ( + CGT_SUPPORT_COPIES_COLUMN, + CGT_SUPPORT_COPY_INDEX_COLUMN, + HOUSEHOLD_IS_CGT_SUPPORT_COPY, +) from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity +from microcosm.build.uk_runtime.high_value_home_support import ( + HIGH_VALUE_SUPPORT_COPIES_COLUMN, + split_high_value_support_households, +) +from microcosm.build.uk_runtime.household_lineage import ( + HIGH_VALUE_SUPPORT_COPY_COLUMN, + HIGH_VALUE_SUPPORT_INDEX_COLUMN, +) from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.receipt_counts import UK_WITHHELD, rounded_count +from microcosm.build.uk_runtime.regional_home_values import ( + assign_regional_home_values, +) +from microcosm.build.uk_runtime.salary_sacrifice import ( + SALSAC_OUTPUT, + SALSAC_PRE_CONVERSION_PAY_COLUMN, +) from microcosm.build.uk_runtime.spi_band_donors import ( HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, PERSON_IS_SPI_INCOME_BAND_CARRIER, @@ -58,8 +84,10 @@ from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate from microcosm.frame import WeightKind, engine_tables from test_support.microcosm_build.uk_clone_first_frames import ( + IdTables, assert_copy_ids_follow_lineage, clone_first_frame, + copy_households, ) from test_support.paths import paths_for @@ -1946,3 +1974,353 @@ def test_e8_receipt_replays_the_seating_of_every_clone() -> None: problems = {} tool._e8_band_donors(_reweighted(stacked, weights), problems, **arguments) assert problems["band_donor_layer"] == {"bands_with_unequal_donor_weights": 1} + + +# The E8 receipt on the artifact as built (microcosm#931). Every layer stacked +# after stage 17 copies rows the stage wrote, donors among them, and mints its +# copies in a copy-id field above the one the donors take. A rerun on a frame +# that kept those rows seated them as households the stage saw, and the donors +# it appended sorted below their ids, which the frame refuses. The receipt +# reruns the stage on the frame the stage wrote, every later row folded onto +# the row it came from. + +#: The copy-id fields minted after stage 17, in the order the spine mints them. +_LATER_FIELDS = ( + "high_value_support", + "cgt_support_split", + "cgt_incidence_clone", + "cgt_residential_split", +) + + +def _stage_17_ids(household: pd.DataFrame) -> np.ndarray: + """Each household's id with every later layer's ordinal cleared.""" + + ids = household["household_id"].to_numpy(dtype=np.int64) + for field in _LATER_FIELDS: + ids = copy_id_table.parent(ids, field) + return ids + + +def _minted(frame, *, field: str, index_column: str, ordinals): + """``frame``'s tables with copies minted in ``field``, and their lineage. + + ``copy_households`` mints every id of a copy through the copy-id table, + as the stages do, and records the copy's ordinal in ``index_column``; + ``ordinals`` maps a household's position to its copies' ordinals. Returns + the tables and, per household, its parent's weight, its ordinal and the + size of its family (one for a household left whole). + """ + + copied = copy_households( + IdTables( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=frame.table("household"), + ), + field=field, + path_column=index_column, + ordinals=ordinals, + rule="table", + ) + ids = copied.household["household_id"].to_numpy(dtype=np.int64) + parents = pd.Series(copy_id_table.parent(ids, field)) + return ( + copied, + _weights_by_id(frame).loc[parents.to_numpy()].to_numpy(), + copy_id_table.ordinal(ids, field), + parents.map(parents.value_counts()).to_numpy(), + ) + + +def _framed(previous, tables, weights): + """``tables`` as a frame at ``weights``, with ``previous``'s mass records.""" + + return uk_national_frame( + person=tables.person, + benunit=tables.benunit, + household=tables.household, + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=weights, + mass_log=previous.mass_log, + ) + + +def _through_the_later_layers(stacked) -> SimpleNamespace: + """Stage 17's output carried through every layer stacked after it. + + The high-value home support (stage 26) and the regional home values (27) + run as built, every household an owner valued on its true source, at a + stratum that divides donors, SPI copies and base households alike. The + CGT layers are replayed by hand with ``copy_households``: the support + split (42) divides the households of one source in four into two or + three rows at equal shares, the incidence twin (43) copies every + household, the anchor (45) moves an eighth of the mass of two pairs in + three, one way or the other, and the residential split (46) divides the + twins of one source in five into two or four arms at product weights. + Salary sacrifice (48) then converts the best-paid record's pay in place. + Every layer keeps each household's mass. All values are invented. + """ + + household = stacked.table("household").copy() + source = household["source_household_id"].to_numpy() + household["tenure_type"] = pd.array( + ["OWNED_OUTRIGHT"] * len(household), dtype="string" + ) + household["main_residence_value"] = 100_000.0 * (1 + source % 10) + household["property_wealth"] = household["main_residence_value"] + 10_000.0 + high_value = split_high_value_support_households( + _with_tables(stacked, household=household), stratum_lower=0.9, copies=3 + ) + valued = assign_regional_home_values(high_value.frame).frame + + household = valued.table("household").copy() + source = household["source_household_id"].to_numpy() + rows = np.where(source % 4 == 1, 2 + (source % 8 == 1), 1) + household[HOUSEHOLD_IS_CGT_SUPPORT_COPY] = False + household[CGT_SUPPORT_COPY_INDEX_COLUMN] = 0 + household[CGT_SUPPORT_COPIES_COLUMN] = rows + tables, weights, ordinal, size = _minted( + _with_tables(valued, household=household), + field="cgt_support_split", + index_column=CGT_SUPPORT_COPY_INDEX_COLUMN, + ordinals={ + int(position): tuple(range(1, int(rows[position]))) + for position in np.flatnonzero(rows > 1) + }, + ) + tables.household[HOUSEHOLD_IS_CGT_SUPPORT_COPY] = ordinal != 0 + split = _framed(valued, tables, weights / size) + + household = split.table("household").copy() + household[HOUSEHOLD_IS_CGT_CLONE] = 0 + tables, weights, ordinal, _ = _minted( + _with_tables(split, household=household), + field="cgt_incidence_clone", + index_column=HOUSEHOLD_IS_CGT_CLONE, + ordinals={position: (1,) for position in range(len(household))}, + ) + tables.household[HOUSEHOLD_IS_CGT_CLONE] = ordinal != 0 + kept = 0.5 + 0.125 * (tables.household["source_household_id"].to_numpy() % 3 - 1) + twinned = _framed(split, tables, weights * np.where(ordinal == 0, kept, 1 - kept)) + + household = twinned.table("household").copy() + source = household["source_household_id"].to_numpy() + gainers = np.where( + household[HOUSEHOLD_IS_CGT_CLONE].to_numpy() & (source % 5 == 0), + 1 + (source % 10 == 0), + 0, + ) + household[HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE] = False + household[CGT_RESIDENTIAL_CLONE_INDEX_COLUMN] = 0 + tables, weights, ordinal, size = _minted( + _with_tables(twinned, household=household), + field="cgt_residential_split", + index_column=CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, + ordinals={ + int(position): tuple(range(1, 2 ** int(gainers[position]))) + for position in np.flatnonzero(gainers) + }, + ) + tables.household[HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE] = ordinal != 0 + # One liable gainer at 1/4, or two at 1/4 and 1/2: arms at product weights. + shares = {1: (1.0,), 2: (0.75, 0.25), 4: (0.375, 0.125, 0.375, 0.125)} + armed = _framed( + twinned, + tables, + weights + * np.asarray([shares[n][j] for n, j in zip(size, ordinal, strict=True)]), + ) + + person = armed.table("person").reset_index(drop=True) + pay = person["employment_income"].to_numpy(dtype=float) + converted = int(np.argmax(pay)) + person["employee_pension_contributions"] = 0.0 + person[SALSAC_OUTPUT] = 0.0 + person[SALSAC_PRE_CONVERSION_PAY_COLUMN] = 0.0 + person.loc[converted, SALSAC_OUTPUT] = pay[converted] / 4 + person.loc[converted, "employment_income"] = pay[converted] * 3 / 4 + person.loc[converted, SALSAC_PRE_CONVERSION_PAY_COLUMN] = pay[converted] + return SimpleNamespace( + artifact=_with_tables(armed, person=person), high_value=high_value + ) + + +def _stage_17_and_artifact(n_clones: int, propensity: pd.DataFrame): + support = ( + _support_frame(200) + if n_clones == 1 + else _clone_first_support_frame(200, n_clones) + ) + stage_17 = stack_spi_income_band_donors( + support, propensity=propensity, **_PUBLISHED + ) + return stage_17, _through_the_later_layers(stage_17.frame) + + +@pytest.mark.parametrize("n_clones", [1, 3]) +def test_e8_receipt_reruns_stage_17_on_the_frame_it_wrote(n_clones: int) -> None: + tool = _identity_tool() + propensity = _propensity() + stage_17, built = _stage_17_and_artifact(n_clones, propensity) + + # Given an artifact whose high-value copies divide donors, SPI copies and + # base households alike, and whose CGT layers copy donors and high-value + # copies too, + divided = built.high_value.frame.table("household") + copies = divided[HIGH_VALUE_SUPPORT_INDEX_COLUMN].to_numpy() != 0 + donor = divided[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR].to_numpy() + spi = divided["household_support_channel"].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL) + assert (copies & donor).any() + assert (copies & spi.to_numpy() & ~donor).any() + assert (copies & ~spi.to_numpy()).any() + household = built.artifact.table("household") + support_copies = household[CGT_SUPPORT_COPY_INDEX_COLUMN].to_numpy() != 0 + for copied in (HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, HIGH_VALUE_SUPPORT_COPY_COLUMN): + assert (support_copies & household[copied].to_numpy()).any() + assert household[HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE].any() + + # its scope to stage 17 is the frame the stage wrote, row for row, at the + # weights it left, + scoped = tool._frame_as_stage_saw(built.artifact, "spi_income_band_donors") + for entity in ("person", "benunit", "household"): + assert sorted(scoped.table(entity)[f"{entity}_id"]) == sorted( + stage_17.frame.table(entity)[f"{entity}_id"] + ) + np.testing.assert_allclose( + _weights_by_id(scoped).loc[_weights_by_id(stage_17.frame).index].to_numpy(), + _weights_by_id(stage_17.frame).to_numpy(), + rtol=1e-12, + ) + + # and the check scales each funding stratum back by the stage's own + # factor, and the stage rerun there reproduces the stored seats, carriers + # and band weights, with the artifact's people in either order and the + # pre-donor people permuted, reading each donor once. + stored = {row["lower_bound"]: row for row in stage_17.band_rows} + factors = {row["stratum"]: row["factor"] for row in stage_17.funding_rows} + for artifact in (built.artifact, tool._reverse_rows(built.artifact)): + problems: dict[str, object] = {} + receipt = tool._e8_band_donors( + artifact, + problems, + propensity=propensity, + band_taxpayers=TAXPAYERS, + permutation_seed=123, + ) + assert problems == {} + assert receipt["seating_recomputed"] is True + assert receipt["donor_households"] == stage_17.donor_count + assert receipt["funding"] == pytest.approx(factors, rel=1e-12) + for row in receipt["bands"]: + band = stored[row["lower_bound"]] + assert row["donor_households"] == band["donor_households"] + assert row["donor_weight"] == pytest.approx(band["donor_weight"], rel=1e-12) + assert row["weighted_taxpayers"] == pytest.approx( + band["weighted_taxpayers"], rel=1e-12 + ) + + +def test_e8_receipt_reads_each_donor_through_its_later_copies() -> None: + tool = _identity_tool() + propensity = _propensity() + _, built = _stage_17_and_artifact(1, propensity) + arguments = { + "propensity": propensity, + "band_taxpayers": TAXPAYERS, + "permutation_seed": 123, + } + # The donor the later layers divided into the most rows: its stage-17 + # row and every row made from it. + household = built.artifact.table("household") + stems = _stage_17_ids(household) + rows = pd.Series(stems).value_counts() + donors = household.loc[ + household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR], "household_id" + ] + donor = min(rows.index[rows.index.isin(donors)], key=lambda s: (-rows[s], s)) + family = np.flatnonzero(stems == donor) + assert len(family) > 2 + weights = _weights_by_id(built.artifact).to_numpy().copy() + + # Mass moved between the rows of a donor's family is no defect: the fold + # gives back the weight stage 17 left. + moved = weights.copy() + moved[family[0]] += moved[family[-1]] / 2 + moved[family[-1]] /= 2 + problems: dict[str, object] = {} + tool._e8_band_donors(_reweighted(built.artifact, moved), problems, **arguments) + assert problems == {} + + # Reweighting the whole family is a defect: its band's donors no longer + # share one weight. + scaled = weights.copy() + scaled[family] *= 1.5 + problems = {} + tool._e8_band_donors(_reweighted(built.artifact, scaled), problems, **arguments) + assert problems["band_donor_layer"] == {"bands_with_unequal_donor_weights": 1} + + +def test_e8_receipt_refuses_a_later_row_its_scope_leaves_in_place() -> None: + # Given an artifact whose high-value copies have lost the lineage the + # scope reads them by, those copies stay in scope; their ids still hold + # an ordinal in a field minted after stage 17, so the receipt refuses by + # count rather than seat them. + tool = _identity_tool() + propensity = _propensity() + _, built = _stage_17_and_artifact(1, propensity) + household = built.artifact.table("household").drop( + columns=[ + HIGH_VALUE_SUPPORT_COPY_COLUMN, + HIGH_VALUE_SUPPORT_INDEX_COLUMN, + HIGH_VALUE_SUPPORT_COPIES_COLUMN, + ] + ) + unread = _with_tables(built.artifact, household=household) + with pytest.raises(ValueError) as caught: + tool._e8_band_donors( + unread, + {}, + propensity=propensity, + band_taxpayers=TAXPAYERS, + permutation_seed=123, + ) + assert str(caught.value) == ( + f"{built.high_value.copies_created} household row(s) left in the scope " + "to the SPI income band donor stage hold an ordinal in a copy-id field " + "minted after it (microcosm#931); the band-donor rerun would seat them " + "as households the stage saw." + ) + + +def test_the_e8_entry_hands_the_band_donor_check_the_artifact(monkeypatch) -> None: + # The band-donor check scopes the artifact itself, which reverses the + # salary-sacrifice conversion; handed the frame the entry has already + # reversed, it would reverse the conversion twice and refuse. The other + # E8 checks need the CGT resources and stand aside here. + from microcosm.build.uk_runtime import cgt_imputation, student_loans + + tool = _identity_tool() + propensity = _propensity() + stage_17, built = _stage_17_and_artifact(1, propensity) + person = built.artifact.table("person").copy() + person["student_loan_plan"] = "NONE" + artifact = _with_tables(built.artifact, person=person) + for check in ("_e8_residential_split", "_e8_clone_pairs", "_e8_support_split"): + monkeypatch.setattr(tool, check, lambda *args, **kwargs: {}) + monkeypatch.setattr( + tool, "_band_donor_seating_inputs", lambda *args: (propensity, TAXPAYERS) + ) + monkeypatch.setattr(cgt_imputation, "uk_cgt_policy_parameters", lambda _: None) + monkeypatch.setattr( + student_loans, + "assign_student_loan_plans", + lambda frame, **kwargs: SimpleNamespace(frame=frame), + ) + + receipt = tool.e8_identity_receipt(artifact, permutation_seed=123) + assert receipt["pre_salary_sacrifice_restored"] is True + assert receipt["stored_column_mismatches"] == {} + assert receipt["band_donors"]["seating_recomputed"] is True + assert receipt["band_donors"]["donor_households"] == stage_17.donor_count diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 5d753a86b9..7f36b441f9 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -1598,22 +1598,33 @@ def _e8_band_donors( weights and their mass left the incumbent households of each donor's region in proportion; where the frame carries geographic clones, the region within the donor's clone, and each clone seats its own donors - (microcosm#931). Folding the CGT layers stacked afterwards gives the - frame the stage wrote; scaling each funding stratum's incumbents back to - the stratum's whole mass gives the weights it read. The stored layer must - agree with itself (one carrier per donor, each parent household copied - once, equal weights within a band, the mass record conserving the total) - and, when the tape's band propensities are supplied (they need the - licensed SPI tape), with the stage's own function rerun on the - reconstructed pre-donor frame: the same source households in the same - bands, the same carriers and the same band weights, in original and - permuted row order. The seating is keyed on person ids, so the rerun - needs no stage-time row order. Without the propensities the receipt says - the seating was not recomputed. Problems land under ``band_donor_*`` keys. + (microcosm#931). ``frame`` is the artifact as built. Stage 17 saw none of + the rows minted after it: the high-value home copies of stage 26 and the + CGT support copies, incidence twins and residential arms of stages 42, + 43 and 46, the households whose ids hold an ordinal in a copy-id field + above ``spi_support_channel``. Dropping them with their people and + benefit units, each one's mass folded back onto the row it came from + (``_frame_as_stage_saw``), gives the frame the stage wrote at the weights + it left, to rounding, and the stored seats, carriers and band weights are + read there, so a later copy of a donor is not read as a donor. That + scoping also reverses the in-place rewrites of ``salary_sacrifice`` and + ``regional_home_values``, so ``frame`` must not be a frame already + scoped or reversed: a second salary-sacrifice reversal refuses. Scaling + each funding stratum's incumbents back to the stratum's whole mass then + gives the weights the stage read. The stored layer must agree with + itself (one carrier per donor, each parent household copied once, equal + weights within a band, the mass record conserving the total) and, when + the tape's band propensities are supplied (they need the licensed SPI + tape), with the stage's own function rerun on the reconstructed + pre-donor frame: the same source households in the same bands, the same + carriers and the same band weights, in original and permuted row order. + The seating is keyed on person ids, so the rerun needs no stage-time row + order. Without the propensities the receipt says the seating was not + recomputed. A row minted after the stage that the scoping leaves in + place is refused by count, since the rerun would seat it as a household + the stage saw. Problems land under ``band_donor_*`` keys. """ - from microcosm.build.uk_runtime.cgt_structure import HOUSEHOLD_IS_CGT_CLONE - from microcosm.build.uk_runtime.cgt_support import HOUSEHOLD_IS_CGT_SUPPORT_COPY from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, @@ -1626,9 +1637,11 @@ def _e8_band_donors( SPI_INCOME_BAND_DONOR_LOWER_BOUNDS, SPI_INCOME_BAND_DONOR_MASS_CHANGE_REASON, SPI_INCOME_BAND_DONOR_SEED, + UK_SPI_INCOME_BAND_DONORS_STAGE_NAME, _funding_strata, stack_spi_income_band_donors, ) + from microcosm.build.uk_runtime.spi_support import SPI_SUPPORT_COPY_ID_FIELD if HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR not in frame.table("household").columns: raise ValueError( @@ -1664,17 +1677,23 @@ def _e8_band_donors( record.declared_factor, ] - # The frame the donor stage wrote: the CGT layers stacked after it folded - # back, so every remaining household carries the weight the stage left. - staged = _drop_stacked_layers( - frame, - [ - flag - for flag in (HOUSEHOLD_IS_CGT_CLONE, HOUSEHOLD_IS_CGT_SUPPORT_COPY) - if flag in frame.table("household").columns - ], - ) + # The frame the donor stage wrote: every row minted after it dropped with + # its people and benefit units, its mass folded back onto the row it came + # from, so every remaining household carries the weight the stage left, + # to rounding. The rerun mints its donors at ordinal 2 of the + # spi_support_channel field, below every later field: a later row left in + # scope would be seated as a household the stage saw, and break the + # frame's ascending ids once the donors are appended after it. + staged = _frame_as_stage_saw(frame, UK_SPI_INCOME_BAND_DONORS_STAGE_NAME) household = staged.table("household") + minted_later = _households_minted_after(SPI_SUPPORT_COPY_ID_FIELD, household) + if minted_later: + raise ValueError( + f"{minted_later} household row(s) left in the scope to the SPI " + "income band donor stage hold an ordinal in a copy-id field minted " + "after it (microcosm#931); the band-donor rerun would seat them as " + "households the stage saw." + ) person = staged.table("person") benunit = staged.table("benunit") weights = np.asarray(staged.weights_for("household").values, dtype=float) @@ -1925,8 +1944,11 @@ def e8_identity_receipt( pre-split frame with the stage's own function - the solved probability carried as arm weights - in original and permuted row order against the stored arms, indices, weights, probabilities and residential gains. The - clone-pair, support-split and band-donor checks run on the frame with the - arms folded onto their sources, the state those layers were written in. + clone-pair and support-split checks run on the frame with the arms folded + onto their sources, the state those layers were written in; the + band-donor check runs on the frame stage 17 wrote, every row minted after + it (the high-value home copies of stage 26 too) folded onto the row it + came from (``_frame_as_stage_saw``). Every CGT layer ran before ``salary_sacrifice``, whose conversion rewrites a converted record's pay in place, so the recomputes first reverse it from the stage's pre-conversion pay carrier @@ -2006,10 +2028,13 @@ def e8_identity_receipt( ) # (4) SPI income band donors: the seating reconstructed from ids, on the - # arm-free frame (the donor stage ran before the residential split). + # frame stage 17 wrote. The check scopes the artifact itself to it + # (``_frame_as_stage_saw``), every layer stacked after stage 17 folded + # away; that scoping reverses the salary-sacrifice conversion on its own, + # so it takes ``frame``, not the reversed ``cgt_frame``. propensity, band_taxpayers = _band_donor_seating_inputs(frame, spi_tab) band_donors = _e8_band_donors( - cgt_frame, + frame, problems, propensity=propensity, band_taxpayers=band_taxpayers, @@ -2619,6 +2644,22 @@ def _require_copy_ids_on_the_declared_table(household: pd.DataFrame) -> None: raise ValueError(f"{off} {_OFF_TABLE_REFUSAL}") +def _households_minted_after(field: str, household: pd.DataFrame) -> int: + """How many household ids hold an ordinal in a copy-id field above ``field``. + + The copy-id table lists its fields in the order the spine mints them, so + the fields above ``field`` are the copy layers stacked after every stage + that mints in it. + """ + + names = [spec.name for spec in copy_id_table.UK_COPY_ID_FIELDS] + ids = _exact_household_ids(household, what="The copy-id field scope") + later = np.zeros(len(ids), dtype=bool) + for name in names[names.index(field) + 1 :]: + later |= copy_id_table.ordinal(ids, name) != 0 + return int(later.sum()) + + def _copy_ordinals_and_parents( ids: np.ndarray, field: str ) -> tuple[np.ndarray, np.ndarray]: From 0982bfa7fc757a19f1692507c3edc60bb3bd32b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:45:37 +0000 Subject: [PATCH 057/108] Fold geographic clones onto their lowest-id record, check record ids by sorting, and add a shared clone expansion core (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared fold in microcosm.build.source_lineage is now fold_replicas, which returns a ReplicaFold with one value per record in each of five fields: kept, the rows the fit trains on; weights, on each kept row its variant's selected weights summed exactly (math.fsum), every other weight as given; representative, the row a selected record's variant trains on, -1 off the selection; varied, whether a selected record's key holds more than one variant; and key_group, a code of its replica key, -1 off the selection. Its repr prints none of them. Each variant trains on its selected record with the least order, ties going to the earlier row, or without order on its first selected row, as before. The early returns are decided on the key. When no two selected records share a key the fold is the identity. When keys repeat but no two selected records share a variant, the selection and weights still come back value for value, and every record of a key with more than one variant is varied. order must hold one numpy integer per record (a timedelta is refused); a malformed one is refused with counts only, at any clone count. collapse_replicas is now a wrapper that returns (kept, weights) and takes the same keywords, order included. A selected record with a null key is now refused: replica_keys never gives one, but the fold used to add such a record to the last key's group, or, with inputs, to the other null-key records that shared its inputs. The UK entry, household_lineage.fold_geographic_clones, returns that fold ordered by the entity's id column, and collapse_geographic_clones keeps its signature and return through it. Within a key the ids differ only by each clone's offset (the copy-id table's geographic_support digits on a clone-first spine), so each variant trains on its lowest selected clone: clone 0 whenever clone 0 is selected and agrees with it, whatever the row order. integer_record_keys in microcosm.build.stochastic_assignment finds a repeated id by comparing neighbours in a sorted copy of the ids instead of building np.unique's array. On integer ids its verdicts, errors and return are unchanged. It now checks the dtype's kind rather than the integer subtype, so timedelta ids, which numpy types as integers and whose NaT never compare equal, are refused as ids that are not integers. The new microcosm.build.clone_expansion is a country-neutral core that turns a frame into K clones of every record, clone-major. It names no country's columns, tables or constants: expand_clones(frame, clones=K, declaration=...) reads every entity, id column, membership column and weighted entity from the frame. The country passes a CloneDeclaration: a CloneIdRule (bound, mint, require_below), a clone-index column for every entity, the weight kinds it accepts, the mass records' reason, and optional source-id columns. A declaration compares and hashes by identity. Clone 0 is the frame's own rows. Clone k's ids are minted by the rule in the ids' own dtype, each person's memberships point at its units' clone k, and every other column is copied unchanged, so ids and memberships keep their dtype at every K. Every row gets its clone ordinal and, where declared, its clone-0 id. Every weighted entity's weights become numpy.tile(w / K, K), of its own kind. The strata repeat, the metadata is kept, and the mass log gains one record per weighted entity, in the frame's entity order, with factor 1.0 and the declared reason, each entity's totals agreeing within a relative 1e-9. At K = 1 the rows, ids and weights are the frame's own. The core refuses: - a clone count that is not an integer in [1, bound]; - a declaration that names an entity the frame lacks, or gives one of its entities no clone-index column; - a frame that already carries a declared column or a link table; - weights, on any entity, of a kind it does not accept; - ids or memberships that are not numpy integers, and memberships in another dtype than their unit's ids; - ids the rule's require_below refuses; - a mint that does not return one integer per id in the ids' own dtype, keep the ids' order or put each clone above the one before. Its refusals name entities, columns, dtypes, weight kinds and declared constants only, and the rule's own refusals must keep to counts and constants. Nothing calls it yet. The geographic expand transform (G1) comes next and will pass the UK's declarations. Its mint will be a named call to copy_id_table inside the UK module, so that the binding rule binds the table to G1's key. The G1 stand-in, clone_first_frame in test_support, does not use the core, so it stays an independent oracle. For Max's review: - The core returns a plain Frame with no lineage. The expand kernel recovers lineage from the ids at the declared radix. - Its per-clone columns are the clone index and an optional source id. A country's labels, such as a support channel, derive from the index. - Memberships follow their unit's minted ids, in the units' id dtype. - require_below reads each entity's id column, since memberships are unit ids. - It splits every weighted entity, each with its own mass record, as decision 10's analysis proposed. A country that weights households only, as the UK does, can refuse other weighted entities in its own wrapper. - It copies no link table and gives every record the same number of clones, until a caller needs more. Why: O8b. A variant trained on whichever of its replicas came first in row order, so the clone a fit trained on depended on how the stages before it ordered their rows. Ordered by id, each variant trains on its lowest-id record whatever the row order. The representative map, variety and key codes are groundwork for the next commits: UC capital's donor count, the fold-variety receipts and the twin check (O9a, O14(i)). O2: as ruled on 10 Nov, the sorted check gives np.unique's verdict without building the unique array, and is faster on large id arrays. Decision 10: María ruled on 10 Nov that the expansion core is shared now, rather than UK-local in G1 until a later extraction, so that another country's clone step can use it. Max reviews it in this PR, beside the shared fold. What moves: node keys only. No value moves at one clone or on any spine. Every spine is one clone today, so each fit's replica keys are unique, the fold returns the selection and weights it was given, value for value, and integer_record_keys gives np.unique's verdict on every integer id array. Only frames whose geographic clones are out of clone order fold differently, and today only tests build them: a variant now trains on its lowest-id clone instead of its first row. These re-key: - every UK spine node, since the spine's root binds stochastic_assignment, source_lineage and household_lineage and keys chain; with them the spine gate phases (the transferred phase's own code also hashes stochastic_assignment) and, in the in-graph build, every node after the spine; - on a bound spine, whose root hashes none of the three, the full build's uk.full.identity, uk.full.property.value and uk.full.private_rent, which hash source_lineage and household_lineage, and every node keyed on them; - the national line's gates, whose registry modules include stochastic_assignment. The spine checkpoint's gate digests do not move. clone_expansion is imported only by its tests, so it binds no node until the geographic expand stage is bound. The derived binding tables are unchanged, since the new names live in modules that are already bound wherever they are reached. No generated file or gates.json changes, since uk_spine.json and the H2 fixture hold stage contract hashes, not code hashes. No transport kernel and no US, NZ or BE node hashes an edited module, so their node keys stay; only the US PUF-support builder's whole-tree resume fingerprint moves, as it does on any edit to packaged source. The comments at the six fold call sites still say the first row; they change with their calls in later commits of this PR. The fits fragment 931-uk-fits-fold-geographic-clones and docs/uk-atomic-household-lineage.md now name fold_replicas as the fold and collapse_replicas as its rows and weights. Tests (engine-free; invented values only): - test_source_lineage.py: - the least order wins under all 720 row orders of a three-clone frame, with and without agreeing inputs; - each variant trains on its own least-order replica, ties going to the earlier row; - a fold without replicas is the identity, with an identity representative map; - representative, varied and key_group match hand-built maps and, on 240 random frames, a record-by-record statement of the contract; - without order the earlier fixtures keep their rows and weights, as the row positions taken as the order do; - a two-clone frame whose clones all differ folds nothing and varies everywhere; - malformed orders, a timedelta among them, and null keys are refused, and the order refusal prints no order value. - test_uk_geographic_clone_collapse.py: with every table in reverse row order, each record keeps clone 0 for households, people and benefit units, and each representative is its key's lowest id. A differing clone trains on its own row, and only its record varies. The scan that holds every folding stage to a live lineage now finds a stage through either UK entry, leaving out the module that defines them. - test_stochastic_assignment.py: the sorted check gives np.unique's verdict on 403 arrays (random ids in five integer dtypes, the int64 extremes, a repeated maximum and an empty array), and 2-D and timedelta ids are refused. - test_clone_expansion.py, which imports no country module, uses neutral declarations at K = 1, 2, 3 and the bound. It checks: - ids, memberships and copied columns; - weights bit for bit against numpy.tile on every weighted entity, each with its own mass record; - the K-free mass record; - the K = 1 identity; - strata and metadata; - the frame handed in left unchanged; - int32 ids and memberships staying int32 at K = 1 and 3; - US-shaped clones, weighted on households and people and labelled from their clone index, passing outer_stage_runtime.assert_clone_expansion; - a declaration hashing by identity; - every refusal, the dtype ones included, with no id or weight in its message. - test_uk_clone_expansion_core.py uses the UK's declarations: copy_id_table's geographic_support mint and require_below, the ladder's clone-index columns, source_household_id and design weights. It checks: - the core equals clone_first_frame bit for bit at K = 1, 2, 3, 7 and 15; - every id decodes to its clone and its clone-0 row; - validate_uk_national_frame passes; - the mass record is the same at every K; - Python strings stay Python strings; - the UK refusals print no id. Mutation checks: - a wrapper that drops the order fails the reverse-order test; - an early return decided on the key and the inputs together fails the all-differ test; - a w * (1 / K) split or an off-by-one clone ordinal fails the UK oracle; - a mint checked by kind alone, or no membership dtype check, fails the dtype tests. These five files, the six fold callers' test files and test_derived_binding_rule.py pass: 370 tests, and one existing skip that needs the licensed SPI donor staged. The 16 binding-rule tests selected from test_uk_graph.py pass. tools/ci_test_plan.py verify is ok, and ruff check and ruff format --check are clean on the nine Python files. Co-Authored-By: Claude Opus 5.5 --- .../931-fold-lowest-id-sorted-keys.changed.md | 21 + .../931-shared-clone-expansion-core.added.md | 28 + ...31-uk-fits-fold-geographic-clones.added.md | 4 +- docs/uk-atomic-household-lineage.md | 2 +- .../src/microcosm/build/clone_expansion.py | 368 ++++++++++ .../src/microcosm/build/source_lineage.py | 178 ++++- .../microcosm/build/stochastic_assignment.py | 10 +- .../build/uk_runtime/household_lineage.py | 54 +- .../shared/test_clone_expansion.py | 691 ++++++++++++++++++ .../engine_free/shared/test_source_lineage.py | 320 ++++++++ .../shared/test_stochastic_assignment.py | 35 + .../uk/test_uk_clone_expansion_core.py | 249 +++++++ .../uk/test_uk_geographic_clone_collapse.py | 97 ++- 13 files changed, 1994 insertions(+), 63 deletions(-) create mode 100644 changelog.d/931-fold-lowest-id-sorted-keys.changed.md create mode 100644 changelog.d/931-shared-clone-expansion-core.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/clone_expansion.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_clone_expansion.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_clone_expansion_core.py diff --git a/changelog.d/931-fold-lowest-id-sorted-keys.changed.md b/changelog.d/931-fold-lowest-id-sorted-keys.changed.md new file mode 100644 index 0000000000..86f3443cc8 --- /dev/null +++ b/changelog.d/931-fold-lowest-id-sorted-keys.changed.md @@ -0,0 +1,21 @@ +The UK fits now train each variant of a survey record on its lowest-id clone instead of its first row, through a shared fold that takes the order from its caller and reports what the clone receipts will need; `integer_record_keys` finds repeated ids by sorting (#931; the sort as María ruled on 10 Nov). No output moves: every spine is one clone today, so every fit keeps its rows and weights value for value, and every id check on integer ids gives the verdict it gave. + +- **`fold_replicas`** (`microcosm.build.source_lineage`) returns a `ReplicaFold` with one value per record in each field: + - `kept`: the rows the fit trains on; + - `weights`: on each kept row, its variant's selected weights summed exactly (`math.fsum`); every other weight unchanged; + - `representative`: the row each selected record's variant trains on, -1 off the selection; + - `varied`: whether a selected record's key holds more than one variant; + - `key_group`: a code of each selected record's replica key, -1 off the selection. + + A variant's representative is its selected record with the least `order`, ties going to the earlier row. `order` is one numpy integer per record, and a malformed one, a timedelta included, is refused with counts only. Without `order` the representative is the variant's first selected record in row order, as before. A selected record with a null key, which `replica_keys` never gives, is now refused rather than folded with other records. +- **Early returns decided on the key.** When no two selected records share a key, the selection and weights come back value for value and nothing varies. When keys repeat but no two selected records share a variant, the selection and weights still come back value for value, and the records of every key with more than one variant vary. Deciding on the key and the inputs together would report no variety exactly when every clone differs. +- **`collapse_replicas`** is a wrapper that returns `fold_replicas`'s kept rows and weights. It takes the same keywords as `fold_replicas`, `order` included. Without `order` a variant keeps its first selected row, as before, so every caller and fixture keeps its values. +- **The UK entry.** `household_lineage.collapse_geographic_clones` keeps its signature and return, and orders the fold by the entity's id column. The new `fold_geographic_clones` returns the whole `ReplicaFold`. A key's clones differ in id only by their clone's offset, so each variant trains on its lowest clone among the selected rows: clone 0 whenever clone 0 is selected and agrees with it, whatever the row order. Before, it trained on whichever of its clones came first. +- **`integer_record_keys`** (`microcosm.build.stochastic_assignment`) compares neighbours in a sorted copy of the ids instead of building `np.unique`'s array. On integer ids its verdicts, errors and return are unchanged. Timedelta ids, which numpy types as integers and the check used to accept, are now refused as ids that are not integers. +- **What moves.** No output, at one clone or on any spine. On frames whose clones are out of clone order, which only tests build today, a variant now trains on its lowest-id record. Node keys move: + - every UK spine node, since the spine's root binds `stochastic_assignment`, `source_lineage` and `household_lineage` and keys chain; with them the spine gate phases (the transferred phase's own code also hashes `stochastic_assignment`) and, in the in-graph build, every node after the spine; + - on a bound spine, whose root hashes none of the three, the full build's `uk.full.identity`, `uk.full.property.value` and `uk.full.private_rent`, which hash `source_lineage` and `household_lineage`, and every node keyed on them; + - the national line's gates, whose registry modules include `stochastic_assignment`. + + The spine checkpoint's gate digests do not move. The derived binding tables are unchanged, since the new names live in modules already bound wherever they are reached. No generated file or `gates.json` changes: `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes. No transport kernel and no US, NZ or BE node hashes an edited module, so their node keys stay; only the US PUF-support builder's whole-tree resume fingerprint moves, as it does on any edit to packaged source. +- **Tests.** The fold is checked against a record-by-record statement of its contract on random frames; under every row order of a three-clone frame; per variant; on a frame without replicas; and on a two-clone frame whose clones all differ, which folds nothing and varies everywhere. Without `order` it keeps the earlier tests' rows and weights. The UK entry keeps clone 0 when every table lists its clones in reverse, and the id check gives `np.unique`'s verdict on random integer arrays of several dtypes and refuses timedeltas. The test that holds every folding stage to a live lineage now finds a stage through either UK entry. diff --git a/changelog.d/931-shared-clone-expansion-core.added.md b/changelog.d/931-shared-clone-expansion-core.added.md new file mode 100644 index 0000000000..410fc8a75c --- /dev/null +++ b/changelog.d/931-shared-clone-expansion-core.added.md @@ -0,0 +1,28 @@ +A country-neutral clone expansion core, `microcosm.build.clone_expansion` (#931; shared now, as María ruled on 10 Nov). `expand_clones(frame, clones=K, declaration=...)` turns a frame into K clones of every record, clone-major. Clone 0 is the frame's own rows with their ids, and clone `k` repeats them with every id minted by the country's rule. The module names no country's columns, tables or constants: it reads each entity, id column, membership column and weighted entity from the frame, and the country passes the rest in a `CloneDeclaration`. + +- **Declarations.** + - A `CloneIdRule`: the most clones the rule numbers (`bound`), `mint(ids, k)` for clone `k`'s ids in the ids' own dtype, and `require_below(ids, what)`, which refuses incoming ids the mint has no room for. + - A clone-index column for every entity. + - The weight kinds the expansion accepts. + - The mass records' reason, declared once, so it is the same at every K. + - Optionally, a column per entity for each row's id before the expansion. + + A declaration compares and hashes by identity. +- **What the clones carry.** + - Each entity's ids are minted for clones 1 to K - 1, and a person's membership in clone `k` is its unit's clone-`k` id. Every other column is copied unchanged. Ids and memberships keep their dtype at every K. + - Each row carries its clone ordinal, with exactly K values on every entity, and, where declared, its clone-0 id. + - Every weighted entity's weights become `numpy.tile(w / K, K)`: one float per row of the frame, the same in each of its clones, of the entity's kind. Strata repeat once per clone and metadata is kept. + - The mass log gains one record per weighted entity, in the frame's entity order, with factor 1.0 and the declared reason. Each entity's clones' total must equal the frame's within a relative 1e-9. + - At K = 1 the rows, ids and weights are the frame's own; only the declared columns and the records are added. +- **Refusals.** Each names entities, columns, dtypes, weight kinds and declared constants only, never an id or a weight. The core refuses: + - a clone count that is not an integer in `[1, bound]`; + - a declaration that names an entity the frame lacks, or gives one of its entities no clone-index column; + - a frame that already carries a declared column, or carries a link table; + - weights, on any entity, of a kind the declaration does not accept; + - ids or memberships that are not numpy integers, memberships in another dtype than their unit's ids, and ids the rule's `require_below` refuses; + - a mint that does not return one integer per id in the ids' own dtype, does not keep the ids' order, or does not put every clone's ids above the clone before's. +- **Not covered yet.** Link tables and a different number of clones per record. These wait for a caller that needs them. +- **Tests.** + - `tests/engine_free/shared/test_clone_expansion.py` runs the core under neutral and US-shaped declarations on invented frames. The US-shaped frame weights households and people, each split with a record of its own, and its clones, labelled from their clone index, pass the shared two-channel invariant `outer_stage_runtime.assert_clone_expansion`. Narrow ids keep their dtype at every K, and each refusal, the dtype ones included, is checked to print no id or weight. + - `tests/engine_free/uk/test_uk_clone_expansion_core.py` runs it under the UK's declarations: the copy-id table's `geographic_support` mint and `require_below`, the ladder's clone-index columns, `source_household_id`, and design weights on households. At K = 1, 2, 3, 7 and 15 it reproduces the clone-first stand-in `clone_first_frame`, which is written without the core, bit for bit on the ids, the household clone index, the source household id and the weights. +- **No caller yet.** No build module imports it yet, so no output and no node key moves. diff --git a/changelog.d/931-uk-fits-fold-geographic-clones.added.md b/changelog.d/931-uk-fits-fold-geographic-clones.added.md index 4cc44ba98e..c83f671259 100644 --- a/changelog.d/931-uk-fits-fold-geographic-clones.added.md +++ b/changelog.d/931-uk-fits-fold-geographic-clones.added.md @@ -1,6 +1,6 @@ The UK fits trained on the frame's FRS rows now fold geographic clones into each survey record before they fit (#931). This covers the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. -- **How the fold works.** Each fit passes what it reads: its predictors and targets, or the columns they derive from. A record's clones that agree on it keep the record's first training row, with the weight of those clones summed exactly. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own, each variant with its own clones' summed weight. SPI, high-value and CGT copies stay rows of their own, as now. +- **How the fold works.** Each fit passes what it reads: its predictors and targets, or the columns they derive from. A record's clones that agree on it keep one training row, the lowest-id one among them (clone 0 whenever clone 0 is selected and agrees, whatever the row order), with the weight of those clones summed exactly. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own, each variant with its own clones' summed weight. SPI, high-value and CGT copies stay rows of their own, as now. - **No change today.** Today's spine has no geographic clones, so every fit's rows and weights stay as they were, value for value. - **What changes once the spine clones before enrichment:** - the fits see each record once wherever its clones agree, and every clone's value at its share wherever they differ; @@ -9,7 +9,7 @@ The UK fits trained on the frame's FRS rows now fold geographic clones into each - **The country-neutral pieces.** `microcosm.build.source_lineage` gains three functions: - `copy_paths`: every household's ordered copy path from a declared lineage, never inferred from ids, weights or values; - `replica_keys`: a key per record of any entity, shared only by its replicas in the named branches, with a member's rank in its household telling members apart; - - `collapse_replicas`: the fold itself, which takes the fit's `inputs` and folds only replicas that agree on them. + - `collapse_replicas`: the fold's training rows and weights (`fold_replicas` returns the whole fold), folding only replicas that agree on the fit's `inputs`. - **The UK entry.** `household_lineage.collapse_geographic_clones` is the UK's entry and requires those inputs. `uk_household_clone_paths` now reads the clone path through `copy_paths` on `UK_COPY_LINEAGE`, with identical paths and draw keys; three identity-test messages name the shared reader's branches. - **Declarations and tests.** - The collapsing stages declare the lineage live where they run, and a test refuses one whose surface would hide it. diff --git a/docs/uk-atomic-household-lineage.md b/docs/uk-atomic-household-lineage.md index d5c10c3f76..1953e1a6d7 100644 --- a/docs/uk-atomic-household-lineage.md +++ b/docs/uk-atomic-household-lineage.md @@ -29,7 +29,7 @@ The UK's copy lineage is declared once, as data, in `uk_runtime.household_lineag - `record_sources` gives every record of any entity its true source through household membership. Every UK floor counts distinct sources through it (`receipt_counts.record_source_households`), so a household's copies and clones count once. - `copy_paths` reads every household's ordered clone path; `uk_household_clone_paths` is the UK's strict entry for the identity kernel. -- `replica_keys` and `collapse_replicas` keep one training row per true source and path without the geographic branch, with the clones' weights summed. The FRS-trained fits fold their training rows through `household_lineage.collapse_geographic_clones`: the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. Once geographic clones precede enrichment, each fit still sees each survey record once. A frame without geographic clones keeps its rows and weights value for value. +- `replica_keys` and `fold_replicas` keep one training row per true source and path without the geographic branch, with the clones' weights summed; through the UK entries each variant trains on its lowest-id clone, whatever the row order. `collapse_replicas` returns the fold's rows and weights alone. The FRS-trained fits fold their training rows through `household_lineage.collapse_geographic_clones`: the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. Once geographic clones precede enrichment, each fit still sees each survey record once. A frame without geographic clones keeps its rows and weights value for value. - Each fit passes what it reads (its predictors and targets, or the columns they derive from), and clones fold only where they agree on it. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own. Each variant carries the summed weight of the clones that share it, so the fit sees every clone's value at its share and nothing is dropped. ## Verification scope diff --git a/packages/microcosm-build/src/microcosm/build/clone_expansion.py b/packages/microcosm-build/src/microcosm/build/clone_expansion.py new file mode 100644 index 0000000000..b46e63ac1e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/clone_expansion.py @@ -0,0 +1,368 @@ +"""Clone-major copies of a frame's records, numbered by a country's rule. + +A build can turn a frame into ``K`` clones of every record. A later stage may +treat each clone as a record of its own (a clone that takes an area of its +own, say), while together the clones stand for the record once, each at an +equal share of its weight. This module makes those clones the same way for +every country (microcosm#931). The country declares how a clone's ids are +numbered (:class:`CloneIdRule`) and what the clones carry +(:class:`CloneDeclaration`). The module names no country's columns, tables or +constants: it reads every entity, id column and membership column from the +frame's :class:`~microcosm.frame.EntitySchema`. + +:func:`expand_clones` returns ``K`` clones of every entity table, +clone-major. Clone 0 is the frame's own rows, ids unchanged. Clone ``k`` +repeats them, with each entity's ids minted by the declared rule, each +person's membership the id of its unit's clone ``k``, and every other column +copied unchanged. Every row carries its clone ordinal, and, where the country +declares a column for it, its id before the expansion. Every weighted +entity's weights are split equally, the same float in each of a row's clones, +and the mass log gains one record per weighted entity stating the declared +reason. + +Every refusal names entities, columns, dtypes, weight kinds and declared +constants only, never an id, a weight or another value of a row. The declared +rule's own refusals must keep to counts and declared constants as well. + +The core copies no link table and gives every record the same number of +clones. Those wait for a caller that needs them. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights + +__all__ = [ + "CLONE_MASS_RTOL", + "CloneDeclaration", + "CloneIdRule", + "expand_clones", +] + +#: How closely the clones' total weight must equal the frame's, relative to +#: the frame's: the tolerance of the Frame's own mass-conserving replacement. +CLONE_MASS_RTOL = 1e-9 + + +@dataclass(frozen=True) +class CloneIdRule: + """How a country numbers its clones' ids. + + ``bound`` is the most clones the rule can number: clone ordinals run + ``0 .. bound - 1``, so a frame expands into 1 to ``bound`` clones. + ``mint(ids, k)`` returns clone ``k``'s ids from the originals', for + ``1 <= k < bound``. It must return one integer per id, of the ids' own + dtype, as a function of that id alone, keep the originals' order, and + put every clone ``k`` id above every clone ``k - 1`` id. A mint in the + ids' dtype keeps it at every clone count. ``require_below(ids, what)`` + refuses incoming ids the mint has no room for; ``what`` names the ids in + its message. + """ + + bound: int + mint: Callable[[np.ndarray, int], np.ndarray] + require_below: Callable[[np.ndarray, str], None] + + def __post_init__(self) -> None: + if not _is_count(self.bound) or self.bound < 1: + raise ValueError("A clone-id rule needs an integer bound of at least 1.") + if not callable(self.mint) or not callable(self.require_below): + raise TypeError( + "A clone-id rule's mint and require_below must be callable." + ) + + +@dataclass(frozen=True, eq=False) +class CloneDeclaration: + """What a country's clones carry, declared once and the same at every K. + + - ``ids``: how the clones' ids are numbered. + - ``clone_index_columns``: each entity's column for its rows' clone + ordinal, ``0 .. K - 1`` as int64. Every entity of the frame needs one. + - ``weight_kinds``: the weight kinds the expansion accepts, on every + entity the frame weights. Each of a row's ``K`` clones takes ``1 / K`` + of its weight, and the clones keep each entity's kind. + - ``reason``: the reason each mass record states. It is declared here, + not per call, so it is the same at every clone count. + - ``source_id_columns``: for any entity, a column for each row's id + before the expansion (its clone-0 id), the same in all its clones. + + A declaration compares and hashes by identity, as its mappings are + frozen views that do not hash. + """ + + ids: CloneIdRule + clone_index_columns: Mapping[str, str] + weight_kinds: frozenset[WeightKind] + reason: str + source_id_columns: Mapping[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not isinstance(self.ids, CloneIdRule): + raise TypeError("A clone declaration's ids must be a CloneIdRule.") + index = _column_names(self.clone_index_columns, "clone-index") + source = _column_names(self.source_id_columns, "source-id") + if not index: + raise ValueError("A clone declaration names a clone-index column.") + columns = [*index.values(), *source.values()] + if len(set(columns)) != len(columns): + raise ValueError("A clone declaration's columns must be distinct.") + kinds = frozenset(self.weight_kinds) + if not kinds or not all(isinstance(kind, WeightKind) for kind in kinds): + raise ValueError("A clone declaration accepts at least one WeightKind.") + if not isinstance(self.reason, str) or not self.reason.strip(): + raise ValueError("A clone declaration states its mass record's reason.") + object.__setattr__(self, "clone_index_columns", MappingProxyType(index)) + object.__setattr__(self, "source_id_columns", MappingProxyType(source)) + object.__setattr__(self, "weight_kinds", kinds) + + +def expand_clones(frame: Frame, *, clones: int, declaration: CloneDeclaration) -> Frame: + """``frame`` as ``clones`` clones of every record, clone-major. + + Each entity table becomes ``clones`` copies of itself, one after another. + Clone 0's rows are the frame's own, with their ids. Clone ``k``'s rows + repeat them: each entity's ids are ``declaration.ids.mint(ids, k)``, and a + person's membership in a unit is that unit's clone ``k`` id. Every other + column is copied unchanged. Each row gains its clone ordinal in its + entity's clone-index column, and its id before the expansion in a + declared source-id column, so a row and its clones share that id. + + Every weighted entity's weights become ``numpy.tile(w / clones, + clones)``: one float per row of the frame, the same in each of its + clones, of the entity's weight kind. The strata repeat once per clone + and the metadata is kept. The mass log gains one record per weighted + entity, in the frame's entity order: the entity's totals before and + after, which agree within :data:`CLONE_MASS_RTOL`, a declared factor of + 1.0, and the declared reason. At ``clones == 1`` the rows, ids and + weights are the frame's own; only the declared columns and the records + are added. + + Refused, naming entities, columns, dtypes, weight kinds and declared + constants only: + + - a clone count that is not an integer in ``[1, bound]``; + - a declaration that names an entity the frame lacks, or gives an entity + of the frame no clone-index column; + - a frame that already carries a declared column or a link table; + - weights, on any entity, of a kind the declaration does not accept; + - an id or membership column that is not of a numpy integer dtype, or a + membership column whose dtype is not its unit ids' dtype; + - incoming ids the rule's ``require_below`` refuses; + - a mint that does not return one integer per id in the ids' own dtype, + does not keep the ids' order, or does not put every clone ``k`` id + above every clone ``k - 1`` id; + - clones whose total weight strays from the frame's beyond + :data:`CLONE_MASS_RTOL`. + """ + + rule = declaration.ids + if not _is_count(clones) or not 1 <= clones <= rule.bound: + raise ValueError( + f"A clone count must be an integer in [1, {rule.bound}], the declared " + "bound." + ) + count = int(clones) + _check_frame(frame, declaration) + schema = frame.schema + ids = {entity: _clone_ids(frame, entity, count, rule) for entity in frame.entities} + weights, records = _split_weights(frame, count, declaration.reason) + ordinals = np.arange(count, dtype=np.int64) + tables: dict[str, pd.DataFrame] = {} + for entity in frame.entities: + table = frame.table(entity) + rows = len(table) + out = table.take(np.tile(np.arange(rows), count)).reset_index(drop=True) + out[schema.entity_id_column(entity)] = np.concatenate(ids[entity]) + if entity == schema.person_entity: + for group in schema.group_entities: + column = schema.membership_column(group) + out[column] = _member_ids(table[column], ids[group]) + source = declaration.source_id_columns.get(entity) + if source is not None: + out[source] = np.tile(ids[entity][0], count) + out[declaration.clone_index_columns[entity]] = np.repeat(ordinals, rows) + tables[entity] = out + return Frame( + tables, + schema, + weights, + pd.concat([frame.strata] * count, ignore_index=True), + mass_log=(*frame.mass_log, *records), + metadata=frame.metadata, + ) + + +def _is_count(value: object) -> bool: + return isinstance(value, int | np.integer) and not isinstance(value, bool) + + +def _column_names(columns: Mapping[str, str], what: str) -> dict[str, str]: + """A declared entity-to-column mapping, each name a non-empty string.""" + + if not isinstance(columns, Mapping): + raise TypeError(f"A clone declaration's {what} columns must be a mapping.") + names = dict(columns) + for entity, column in names.items(): + if not ( + isinstance(entity, str) and entity and isinstance(column, str) and column + ): + raise ValueError( + f"A clone declaration's {what} columns map entity names to " + "column names." + ) + return names + + +def _check_frame(frame: Frame, declaration: CloneDeclaration) -> None: + """Refuse a frame the declaration cannot clone, before anything is minted.""" + + entities = frame.entities + named = {*declaration.clone_index_columns, *declaration.source_id_columns} + unknown = sorted(named.difference(entities)) + if unknown: + raise ValueError( + f"The clone declaration names entities the frame lacks: {unknown}." + ) + unindexed = [e for e in entities if e not in declaration.clone_index_columns] + if unindexed: + raise ValueError( + f"The clone declaration gives {unindexed} no clone-index column." + ) + declared = { + *declaration.clone_index_columns.values(), + *declaration.source_id_columns.values(), + } + carried = set().union(*(frame.table(e).columns for e in entities)) + held = sorted(declared & carried) + if held: + raise ValueError(f"The frame already carries the declared column(s) {held}.") + if frame.links: + raise ValueError("The clone expansion copies no link table; the frame has one.") + for entity in frame.weighted_entities: + kind = frame.weights_for(entity).kind + if kind not in declaration.weight_kinds: + accepted = sorted(k.value for k in declaration.weight_kinds) + raise ValueError( + f"The clone declaration accepts {accepted} weights, not " + f"{kind.value!r} weights on {entity!r}." + ) + schema = frame.schema + person = frame.table(schema.person_entity) + columns = [(frame.table(e), schema.entity_id_column(e)) for e in entities] + columns += [(person, schema.membership_column(g)) for g in schema.group_entities] + for table, column in columns: + dtype = table[column].dtype + if not (isinstance(dtype, np.dtype) and dtype.kind in "iu"): + raise TypeError( + f"The clone expansion mints numpy integer ids; {column!r} holds " + f"{dtype}." + ) + # A membership points at its unit's minted ids, so it must share their + # dtype, or a clone count above one would change it. + for group in schema.group_entities: + membership, unit = schema.membership_column(group), schema.id_column(group) + found, wanted = person[membership].dtype, frame.table(group)[unit].dtype + if found != wanted: + raise TypeError( + f"The clone expansion needs {membership!r} in the dtype of " + f"{unit!r}; it holds {found} against {wanted}." + ) + for entity in entities: + column = schema.entity_id_column(entity) + declaration.ids.require_below( + frame.table(entity)[column].to_numpy(copy=True), f"{column} values" + ) + + +def _clone_ids( + frame: Frame, entity: str, count: int, rule: CloneIdRule +) -> list[np.ndarray]: + """``entity``'s ids in each clone: the frame's, then each clone's minted. + + The frame's ids are distinct (and a group's ascend), so a mint that keeps + their order strictly and puts each clone above the one before gives ids + that are distinct across all clones and ascend in every group table. + """ + + column = frame.schema.entity_id_column(entity) + original = frame.table(entity)[column].to_numpy(copy=True) + original.setflags(write=False) + order = np.argsort(original, kind="stable") + clones = [original] + for k in range(1, count): + minted = np.asarray(rule.mint(original, k)) + # The frame's ids are numpy integers, so a mint in their dtype mints + # integers, and concatenating the clones keeps the ids' dtype. + if minted.shape != original.shape or minted.dtype != original.dtype: + raise ValueError( + f"The clone-id rule must mint one integer for each {entity} id, " + "in the ids' own dtype." + ) + ranked = minted[order] + if (ranked[1:] <= ranked[:-1]).any(): + raise ValueError( + f"The clone-id rule must keep the order of the {entity} ids it copies." + ) + if minted.size and int(minted.min()) <= int(clones[-1].max()): + raise ValueError( + f"The clone-id rule must mint every clone's {entity} ids above " + "the clone before's." + ) + clones.append(minted) + return clones + + +def _member_ids(members: pd.Series, unit_ids: list[np.ndarray]) -> np.ndarray: + """A membership column in every clone: clone ``k``'s members point at + their unit's clone ``k`` id. + + A frame's group ids ascend and are exactly the ids its members name, so + each member's position in its unit's table is exact. + """ + + values = members.to_numpy(copy=True) + positions = np.searchsorted(unit_ids[0], values) + return np.concatenate([values, *(clone[positions] for clone in unit_ids[1:])]) + + +def _equal_shares(values: np.ndarray, count: int) -> np.ndarray: + """Each row's weight split into ``count`` equal shares, clone-major.""" + + return np.tile(values / count, count) + + +def _split_weights( + frame: Frame, count: int, reason: str +) -> tuple[dict[str, Weights], tuple[MassChangeRecord, ...]]: + """Every weighted entity's split weights, and the mass record each leaves, + in the frame's entity order.""" + + weights: dict[str, Weights] = {} + records: list[MassChangeRecord] = [] + for entity in frame.weighted_entities: + before = frame.weights_for(entity) + after = Weights(_equal_shares(before.values, count), before.kind) + if not np.isclose(after.total, before.total, rtol=CLONE_MASS_RTOL, atol=0.0): + raise ValueError( + f"The clones' {entity} weights stray from the frame's total mass " + f"beyond the relative tolerance {CLONE_MASS_RTOL}." + ) + weights[entity] = after + records.append( + MassChangeRecord( + entity=entity, + old_total=before.total, + new_total=after.total, + declared_factor=1.0, + reason=reason, + ) + ) + return weights, tuple(records) diff --git a/packages/microcosm-build/src/microcosm/build/source_lineage.py b/packages/microcosm-build/src/microcosm/build/source_lineage.py index b3b48265cc..b150893956 100644 --- a/packages/microcosm-build/src/microcosm/build/source_lineage.py +++ b/packages/microcosm-build/src/microcosm/build/source_lineage.py @@ -19,18 +19,22 @@ declared copy column either. - :func:`copy_paths` gives every household its ordered copy path, read from the declared columns, never inferred from ids, weights or values. -- :func:`replica_keys` and :func:`collapse_replicas` keep one training row per +- :func:`replica_keys` and :func:`fold_replicas` keep one training row per true source and path without the named branches, with the rows' weights summed, so a fit trained on the frame sees each survey record once however - many geographic clones carry it. A frame without such replicas keeps its - rows and weights value for value. + many geographic clones carry it. The row kept is the replica that comes + first in the caller's order (its lowest id, say). :class:`ReplicaFold` + also says which row each record trains on, which records' replicas differ + and which records share a key; :func:`collapse_replicas` returns its rows + and weights alone. A frame without such replicas keeps its rows and + weights value for value. """ from __future__ import annotations import math from collections.abc import Iterable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field import numpy as np import pandas as pd @@ -41,8 +45,10 @@ __all__ = [ "CopyBranch", "CopyLineage", + "ReplicaFold", "collapse_replicas", "copy_paths", + "fold_replicas", "record_sources", "replica_keys", ] @@ -329,30 +335,71 @@ def replica_keys( ) -def collapse_replicas( +@dataclass(frozen=True, eq=False) +class ReplicaFold: + """A fit's training rows, each record's replicas folded in. + + Every field holds one value per record, in row order: + + - ``kept``: the rows the fit trains on, one per variant of each key; + - ``weights``: on a kept row, the selected weights of its variant summed + exactly; every other weight as it was given; + - ``representative``: the row a selected record's variant trains on, and + -1 off the selection; + - ``varied``: whether a selected record's key holds more than one + variant, so that its replicas did not all fold into one row; + - ``key_group``: a code of a selected record's replica key, shared by + every selected record of that key, and -1 off the selection. + + The fields carry per-record weights, so the representation prints none. + """ + + kept: np.ndarray = field(repr=False) + weights: np.ndarray = field(repr=False) + representative: np.ndarray = field(repr=False) + varied: np.ndarray = field(repr=False) + key_group: np.ndarray = field(repr=False) + + +def fold_replicas( keys: np.ndarray, rows: np.ndarray, weights: np.ndarray, *, inputs: pd.DataFrame | None = None, -) -> tuple[np.ndarray, np.ndarray]: - """Keep the first of each key's ``rows``, carrying the key's summed weight. + order: np.ndarray | None = None, +) -> ReplicaFold: + """Fold each key's selected replicas into one row per variant. - ``rows`` marks the records a fit trains on and ``weights`` their weights, - both over every record. The first selected record of each key, in row - order, stays selected with the selected weights of its key summed - exactly (:func:`math.fsum`, so the sum does not depend on row order); its - replicas leave the selection. Weights off the selection are returned - unchanged. When no two selected records share a key, the selection and - weights are returned as they were, value for value. + ``keys`` gives every record its replica key (:func:`replica_keys`), + ``rows`` marks the records a fit trains on and ``weights`` their + weights, all three over every record. The selected records that share a + key are one record's replicas. ``inputs`` holds, one row per record, what the fit reads: its predictors - and targets, or the columns they derive from. Replicas then fold only - where they agree on every one of those columns. Replicas that differ stay - rows of their own: each distinct variant keeps its first record, with - the summed weight of the replicas that share it, so the fit sees every - variant at its share and nothing is dropped (microcosm#931). A missing - value agrees only with a missing value. + and targets, or the columns they derive from. Replicas fold only where + they agree on every one of those columns, a missing value agreeing only + with a missing value. Replicas that agree on them form one variant of + the record, and each variant folds on its own, so the fit sees every + variant at its share and nothing is dropped (microcosm#931). Without + ``inputs`` a key is one variant. + + Each variant trains on its representative: its selected record with the + least ``order``, ties going to the earlier row, or without ``order`` its + first selected record in row order. ``order`` holds one integer per + record, each record's id say, so that the replica a variant trains on + does not depend on the row order. The representative carries its + variant's selected weights summed exactly (:func:`math.fsum`, so the sum + does not depend on row order either), and the variant's other records + leave the selection. Every other weight is returned unchanged. + + Variety is decided per key. When no two selected records share a key, + the selection and weights come back as they were, value for value, each + selected record its own representative and none varied. When keys + repeat but no two selected records share a variant, the selection and + weights come back as they were too, and every record of a key with more + than one variant is varied: replicas that all differ fold nothing, yet + their record varies. """ keys = np.asarray(keys, dtype=object) @@ -360,17 +407,37 @@ def collapse_replicas( values = np.asarray(weights, dtype=np.float64) if not (len(keys) == len(selected) == len(values)): raise ValueError("Replica keys, rows and weights must align.") - positions = np.flatnonzero(selected) - if inputs is None: - codes, uniques = pd.factorize( - pd.Series(keys[positions], dtype=object), sort=False - ) - groups = len(uniques) - else: + ranks = None + if order is not None: + ranks = np.asarray(order) + # Timedeltas subclass numpy's integers but are not ids, so the kind + # is checked, not the subtype. + if ranks.shape != (len(keys),) or ranks.dtype.kind not in "iu": + raise ValueError( + "Replica order must be one integer per record: " + f"{ranks.size} value(s) for {len(keys)} record(s)." + ) + if inputs is not None: if not isinstance(inputs, pd.DataFrame) or len(inputs) != len(keys): raise ValueError("Replica inputs must be a frame with one row per record.") if not inputs.columns.is_unique: raise ValueError("Replica inputs must name each column once.") + positions = np.flatnonzero(selected) + key_codes, key_uniques = pd.factorize( + pd.Series(keys[positions], dtype=object), sort=False + ) + if (key_codes < 0).any(): + raise ValueError("Replica keys must not be null on a selected record.") + key_group = np.full(len(keys), -1, dtype=np.int64) + key_group[positions] = key_codes + representative = np.full(len(keys), -1, dtype=np.int64) + varied = np.zeros(len(keys), dtype=bool) + if len(key_uniques) == len(positions): + representative[positions] = positions + return ReplicaFold(selected, values, representative, varied, key_group) + if inputs is None: + codes, groups = key_codes, len(key_uniques) + else: table = inputs.iloc[positions].reset_index(drop=True) table.columns = [f"input:{index}" for index in range(table.shape[1])] for column in table.columns: @@ -382,17 +449,52 @@ def collapse_replicas( .ngroup() .to_numpy() ) - groups = int(codes.max()) + 1 if len(codes) else 0 + groups = int(codes.max()) + 1 + # A variant lies inside one key, so a key's variants are counted by the + # key each variant code belongs to. + variant_key = np.empty(groups, dtype=np.int64) + variant_key[codes] = key_codes + variants = np.bincount(variant_key, minlength=len(key_uniques)) + varied[positions] = variants[key_codes] > 1 if groups == len(positions): - return selected, values - first = np.full(groups, -1, dtype=np.int64) - members: list[list[float]] = [[] for _ in range(groups)] - for position, code in zip(positions.tolist(), codes.tolist(), strict=True): - if first[code] < 0: - first[code] = position - members[code].append(float(values[position])) - kept = np.zeros(len(selected), dtype=bool) + representative[positions] = positions + return ReplicaFold(selected, values, representative, varied, key_group) + # Each variant's selected records by least order, then row; the first of + # each run is its representative. Every code from 0 to groups - 1 occurs, + # so the runs come in code order. + if ranks is None: + sequence = np.lexsort((positions, codes)) + else: + sequence = np.lexsort((positions, ranks[positions], codes)) + runs = np.flatnonzero(np.diff(codes[sequence], prepend=-1)) + first = positions[sequence[runs]] + representative[positions] = first[codes] + kept = np.zeros(len(keys), dtype=bool) kept[first] = True summed = values.copy() - summed[first] = [math.fsum(group) for group in members] - return kept, summed + summed[first] = [ + math.fsum(variant) + for variant in np.split(values[positions[sequence]], runs[1:]) + ] + return ReplicaFold(kept, summed, representative, varied, key_group) + + +def collapse_replicas( + keys: np.ndarray, + rows: np.ndarray, + weights: np.ndarray, + *, + inputs: pd.DataFrame | None = None, + order: np.ndarray | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """The training rows and weights of :func:`fold_replicas`, and nothing else. + + Returns ``(kept, weights)`` over every record: each variant's + representative stays selected with its variant's summed weight, and + without ``order`` the representative is its first selected record in row + order. A selection in which no two records share a variant comes back as + it was, value for value. + """ + + fold = fold_replicas(keys, rows, weights, inputs=inputs, order=order) + return fold.kept, fold.weights diff --git a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py index c76bb778e1..a4894ba5bc 100644 --- a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py +++ b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py @@ -136,13 +136,17 @@ def integer_record_keys( The key string formats each id, so the integer ``1001`` and the float ``1001.0`` draw differently, and a merge that turned an id column into floats would silently redraw every record. A repeated id would give two - records the same uniforms. + records the same uniforms. Repeats are found on a sorted copy, where + equal ids sit side by side, without building the unique ids. """ keys = np.asarray(ids) - if keys.ndim != 1 or not np.issubdtype(keys.dtype, np.integer): + # Timedeltas subclass numpy's integers but are not ids, and repeated NaT + # never compare equal, so the kind is checked, not the subtype. + if keys.ndim != 1 or keys.dtype.kind not in "iu": raise TypeError(f"{label} must be a one-dimensional array of integer ids.") - if np.unique(keys).size != keys.size: + ordered = np.sort(keys) + if (ordered[1:] == ordered[:-1]).any(): raise ValueError(f"{label} must name each record once.") return keys diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py index 2adfc71827..e3f38a4da1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py @@ -21,8 +21,9 @@ from microcosm.build.source_lineage import ( CopyBranch, CopyLineage, - collapse_replicas, + ReplicaFold, copy_paths, + fold_replicas, replica_keys, ) from microcosm.build.uk_runtime.national_frame import UK_NATIONAL_SCHEMA @@ -35,6 +36,7 @@ "UK_COPY_LINEAGE", "UK_HOUSEHOLD_LINEAGE_COLUMNS", "collapse_geographic_clones", + "fold_geographic_clones", "uk_household_clone_paths", ] @@ -154,30 +156,37 @@ def uk_household_clone_paths( return [int(source) for source in sources], paths -def collapse_geographic_clones( +def fold_geographic_clones( tables: Mapping[str, pd.DataFrame], entity: str, rows: np.ndarray, weights: np.ndarray, *, inputs: pd.DataFrame, -) -> tuple[np.ndarray, np.ndarray]: +) -> ReplicaFold: """A fit's training rows with each record's geographic clones folded in. - A survey record a fit trains on keeps one row, its first among ``rows``, - with the weight of all its geographic clones; every other copy (SPI - support, high-value support and the CGT steps) stays a row of its own - (microcosm#931). The record's key is its true source and its clone path - without the geographic branch, through the shared reader + A survey record a fit trains on keeps one row, its clone with the lowest + id among ``rows``, with the weight of all its geographic clones; every + other copy (SPI support, high-value support and the CGT steps) stays a + row of its own (microcosm#931). The record's key is its true source and + its clone path without the geographic branch, through the shared reader (``microcosm.build.source_lineage``); a frame without geographic clones, as every spine is today, keeps ``rows`` and ``weights`` value for value. + The fold orders a key's clones by ``entity``'s id column. Their ids + differ only by each clone's offset (the copy-id table's + ``geographic_support`` digits on a clone-first spine), so each variant + trains on its lowest clone among ``rows``: clone 0 whenever clone 0 is + selected and agrees with it, whatever the row order. + ``inputs`` is what the fit reads, one row per row of ``entity``. Clones fold only where they agree on it. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own, - each variant with its clones' summed weight, so the fit sees each - clone's value at its share. Returns the kept rows and the weights over - every row of ``entity``. + each variant on its lowest-id clone with its clones' summed weight, so + the fit sees each clone's value at its share. Returns the shared + :class:`~microcosm.build.source_lineage.ReplicaFold` over every row of + ``entity``. """ keys = replica_keys( @@ -187,7 +196,28 @@ def collapse_geographic_clones( UK_COPY_LINEAGE, merge=(GEOGRAPHIC_SUPPORT_BRANCH,), ) - return collapse_replicas(keys, rows, weights, inputs=inputs) + ids = tables[entity][UK_NATIONAL_SCHEMA.entity_id_column(entity)].to_numpy() + return fold_replicas(keys, rows, weights, inputs=inputs, order=ids) + + +def collapse_geographic_clones( + tables: Mapping[str, pd.DataFrame], + entity: str, + rows: np.ndarray, + weights: np.ndarray, + *, + inputs: pd.DataFrame, +) -> tuple[np.ndarray, np.ndarray]: + """The kept rows and weights of :func:`fold_geographic_clones`. + + Each survey record trains on its lowest-id clone among ``rows``, with + the weight of the clones that agree with it on ``inputs``; clones that + differ stay rows of their own. Returns the kept rows and the weights + over every row of ``entity``. + """ + + fold = fold_geographic_clones(tables, entity, rows, weights, inputs=inputs) + return fold.kept, fold.weights def without_geographic_branch( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_clone_expansion.py b/packages/microcosm-build/tests/engine_free/shared/test_clone_expansion.py new file mode 100644 index 0000000000..d5aea2edbb --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_clone_expansion.py @@ -0,0 +1,691 @@ +"""The shared clone expansion core, on invented frames (microcosm#931). + +Neutral and US-shaped declarations only; no country module is imported. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build import clone_expansion +from microcosm.build.clone_expansion import ( + CLONE_MASS_RTOL, + CloneDeclaration, + CloneIdRule, + expand_clones, +) +from microcosm.build.outer_stage_runtime import assert_clone_expansion +from microcosm.frame import ( + EntitySchema, + Frame, + LinkSpec, + MassChangeRecord, + WeightKind, + Weights, +) + +SCHEMA = EntitySchema(group_entities=("unit", "household")) +#: An invented id field: clone ``k``'s ids are the originals' plus ``k`` times it. +RADIX = 100_000 +BOUND = 5 +REASON = "Clones split each household's weight equally; the mass is conserved." +INDEX_COLUMNS = { + "person": "person_clone", + "unit": "unit_clone", + "household": "household_clone", +} +SOURCE_COLUMNS = {"household": "household_origin"} +DECLARED_COLUMNS = (*INDEX_COLUMNS.values(), *SOURCE_COLUMNS.values()) +HOUSEHOLD_WEIGHTS = np.array([312.5, 47.25, 1009.75]) +PRIOR_RECORD = MassChangeRecord( + entity="household", + old_total=1000.0, + new_total=1369.5, + declared_factor=None, + reason="an earlier stage's declared change", +) + + +def _mint(ids: np.ndarray, k: int) -> np.ndarray: + return ids + k * RADIX + + +def _require_below(ids: np.ndarray, what: str) -> None: + reached = int((np.asarray(ids) >= RADIX).sum()) + if reached: + raise ValueError(f"{reached} of the {what} reach the clone field.") + + +RULE = CloneIdRule(bound=BOUND, mint=_mint, require_below=_require_below) + + +def _declaration(**changes: object) -> CloneDeclaration: + fields: dict[str, object] = { + "ids": RULE, + "clone_index_columns": INDEX_COLUMNS, + "weight_kinds": frozenset({WeightKind.DESIGN}), + "reason": REASON, + "source_id_columns": SOURCE_COLUMNS, + } + fields.update(changes) + return CloneDeclaration(**fields) # type: ignore[arg-type] + + +DECLARATION = _declaration() + + +def _tables(households: tuple[int, int, int] = (411, 522, 633)) -> dict: + """Three invented households: the first holds two units and three people. + + The person rows are out of id order, so a mint must keep the order it is + handed rather than an ascending one. + """ + + first, second, third = households + person = pd.DataFrame( + { + "person_id": np.array([52201, 41101, 63302, 41102, 63301, 41103]), + "person_unit_id": np.array([5221, 4111, 6331, 4111, 6331, 4112]), + "person_household_id": np.array( + [second, first, third, first, third, first] + ), + "pay": [1250.5, 0.0, 87.25, 3020.0, 415.75, 12.5], + "age": np.array([44, 71, 9, 38, 36, 15], dtype=np.int16), + "role": pd.Series(["a", "b", "c", "b", "a", "c"], dtype=object), + "works": [True, False, False, True, True, False], + } + ) + unit = pd.DataFrame( + { + "unit_id": np.array([4111, 4112, 5221, 6331]), + "unit_kind": pd.Categorical(["couple", "single", "single", "couple"]), + } + ) + household = pd.DataFrame( + { + "household_id": np.array([first, second, third]), + "area": ["north", "south", "north"], + } + ) + return {"person": person, "unit": unit, "household": household} + + +def _frame( + tables: dict | None = None, + *, + kind: WeightKind = WeightKind.DESIGN, + schema: EntitySchema = SCHEMA, + extra_weights: dict | None = None, +) -> Frame: + return Frame( + _tables() if tables is None else tables, + schema, + {"household": Weights(HOUSEHOLD_WEIGHTS, kind), **(extra_weights or {})}, + pd.Series(["s1", "s2", "s1", "s2", "s1", "s2"], dtype=object), + mass_log=(PRIOR_RECORD,), + metadata={"period": "2030"}, + ) + + +def _ids_and_weights(frame: Frame) -> set[str]: + """Every id, membership and weight of ``frame``, as text.""" + + texts = { + repr(float(value)) + for entity in frame.weighted_entities + for value in frame.weights_for(entity).values + } + for entity in frame.entities: + table = frame.table(entity) + for column in table.columns: + if column.endswith("_id"): + texts.update(str(value) for value in table[column].tolist()) + return texts + + +def _assert_names_no_value(error: pytest.ExceptionInfo, frame: Frame) -> None: + message = str(error.value) + assert not {text for text in _ids_and_weights(frame) if text in message}, message + + +@pytest.mark.parametrize("clones", [1, 2, 3, BOUND]) +def test_each_clone_repeats_the_frame_with_its_ids_minted(clones: int) -> None: + frame = _frame() + expanded = expand_clones(frame, clones=clones, declaration=DECLARATION) + for entity in frame.entities: + before = frame.table(entity).reset_index(drop=True) + after = expanded.table(entity) + rows = len(before) + assert len(after) == clones * rows + id_column = SCHEMA.entity_id_column(entity) + ids = before[id_column].to_numpy() + assert after[id_column].tolist() == [ + value + k * RADIX for k in range(clones) for value in ids.tolist() + ] + index = after[INDEX_COLUMNS[entity]] + assert index.dtype == np.int64 + assert index.tolist() == [k for k in range(clones) for _ in range(rows)] + assert sorted(set(index.tolist())) == list(range(clones)) + # Every other column is the frame's, repeated once per clone. + copied = [c for c in before.columns if c != id_column and "_id" not in c] + pd.testing.assert_frame_equal( + after[copied], + pd.concat([before[copied]] * clones, ignore_index=True), + check_exact=True, + ) + person = expanded.table("person") + original = frame.table("person") + for group in SCHEMA.group_entities: + column = SCHEMA.membership_column(group) + assert person[column].tolist() == [ + value + k * RADIX + for k in range(clones) + for value in original[column].tolist() + ] + # A member of clone k belongs to its unit's clone k. + units = expanded.table(group).set_index(SCHEMA.id_column(group)) + assert np.array_equal( + units.loc[person[column], INDEX_COLUMNS[group]].to_numpy(), + person[INDEX_COLUMNS["person"]].to_numpy(), + ) + household = expanded.table("household") + origin = household[SOURCE_COLUMNS["household"]] + assert origin.dtype == np.int64 + assert origin.tolist() == [411, 522, 633] * clones + assert SOURCE_COLUMNS["household"] not in person.columns + + +def test_a_numpy_integer_clone_count_is_accepted() -> None: + expanded = expand_clones(_frame(), clones=np.int64(2), declaration=DECLARATION) + assert expanded.n("household") == 6 + + +@pytest.mark.parametrize("clones", [1, 2, 3, BOUND]) +def test_each_row_s_clones_carry_one_equal_share_of_its_weight(clones: int) -> None: + frame = _frame() + weights = expand_clones(frame, clones=clones, declaration=DECLARATION).weights_for( + "household" + ) + assert weights.kind is WeightKind.DESIGN + assert ( + weights.values.tobytes() + == np.tile(HOUSEHOLD_WEIGHTS / clones, clones).tobytes() + ) + by_clone = weights.values.reshape(clones, len(HOUSEHOLD_WEIGHTS)) + assert all(np.array_equal(row, by_clone[0]) for row in by_clone) + assert np.isclose( + weights.total, HOUSEHOLD_WEIGHTS.sum(), rtol=CLONE_MASS_RTOL, atol=0.0 + ) + + +def test_the_mass_record_states_the_declared_reason_at_every_clone_count() -> None: + frame = _frame() + before = frame.weights_for("household").total + records = [] + for clones in range(1, BOUND + 1): + expanded = expand_clones(frame, clones=clones, declaration=DECLARATION) + assert expanded.mass_log[:-1] == (PRIOR_RECORD,) + records.append(expanded.mass_log[-1]) + assert records[-1].new_total == expanded.weights_for("household").total + assert {record.reason for record in records} == {REASON} + assert {record.entity for record in records} == {"household"} + assert {record.declared_factor for record in records} == {1.0} + assert {record.old_total for record in records} == {before} + + +def test_one_clone_keeps_the_frame_s_rows_ids_and_weights() -> None: + frame = _frame() + expanded = expand_clones(frame, clones=1, declaration=DECLARATION) + for entity in frame.entities: + added = [c for c in expanded.table(entity).columns if c in DECLARED_COLUMNS] + # Appended after the frame's own columns: the source id, then the index. + assert added == [ + c + for c in (SOURCE_COLUMNS.get(entity), INDEX_COLUMNS[entity]) + if c is not None + ] + assert list(expanded.table(entity).columns[-len(added) :]) == added + pd.testing.assert_frame_equal( + expanded.table(entity).drop(columns=added), + frame.table(entity).reset_index(drop=True), + check_exact=True, + ) + assert ( + expanded.weights_for("household").values.tobytes() + == HOUSEHOLD_WEIGHTS.tobytes() + ) + + +def test_every_weighted_entity_is_split_with_a_record_of_its_own() -> None: + person_weights = np.array([2.5, 0.75, 4.0, 1.25, 3.5, 0.5]) + frame = _frame(extra_weights={"person": Weights(person_weights, WeightKind.DESIGN)}) + expanded = expand_clones(frame, clones=3, declaration=DECLARATION) + split = {"person": person_weights, "household": HOUSEHOLD_WEIGHTS} + assert expanded.weighted_entities == tuple(split) + for entity, values in split.items(): + weights = expanded.weights_for(entity) + assert weights.kind is WeightKind.DESIGN + assert weights.values.tobytes() == np.tile(values / 3, 3).tobytes() + # One record per weighted entity, in the frame's entity order. + assert expanded.mass_log[:-2] == (PRIOR_RECORD,) + records = expanded.mass_log[-2:] + assert [record.entity for record in records] == list(split) + for record in records: + before = Weights(split[record.entity], WeightKind.DESIGN) + assert record.old_total == before.total + assert record.new_total == expanded.weights_for(record.entity).total + assert (record.declared_factor, record.reason) == (1.0, REASON) + + +def test_the_strata_repeat_and_the_metadata_is_kept() -> None: + frame = _frame() + expanded = expand_clones(frame, clones=3, declaration=DECLARATION) + assert expanded.strata.tolist() == frame.strata.tolist() * 3 + assert dict(expanded.metadata) == dict(frame.metadata) + + +def test_the_frame_handed_in_is_left_as_it_was() -> None: + frame = _frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + expand_clones(frame, clones=BOUND, declaration=DECLARATION) + for entity, table in tables.items(): + pd.testing.assert_frame_equal(frame.table(entity), table, check_exact=True) + assert ( + frame.weights_for("household").values.tobytes() == HOUSEHOLD_WEIGHTS.tobytes() + ) + assert frame.mass_log == (PRIOR_RECORD,) + + +# --------------------------------------------------------------------------- +# US-shaped declarations: the shared two-channel invariant accepts the core's +# clones once the channel labels are written from the clone index. +# --------------------------------------------------------------------------- + +US_SHAPED_SCHEMA = EntitySchema( + group_entities=("household", "tax_unit", "spm_unit", "family", "marital_unit") +) +CHANNELS = ("asec", "puf") +US_SHAPED_MULTIPLIER = 1_000_000 +US_SHAPED_HOUSEHOLD_WEIGHTS = np.array([4.0, 6.5, 2.25]) +US_SHAPED_PERSON_WEIGHTS = np.array([3.5, 4.5, 6.25, 2.0, 2.5]) + + +def _us_shaped_frame() -> Frame: + groups = US_SHAPED_SCHEMA.group_entities + person = pd.DataFrame( + { + "person_id": np.array([101, 102, 201, 301, 302]), + **{ + US_SHAPED_SCHEMA.membership_column(group): np.array( + [1, 1, 2, 3, 3] if group == "household" else [11, 11, 21, 31, 32] + ) + for group in groups + }, + "source_year": np.array([2023, 2023, 2024, 2024, 2024], dtype=np.int16), + "source_household_id": np.array([7, 7, 9, 12, 12]), + "source_person_id": pd.Series(["a", "b", "c", "d", "e"], dtype=object), + "source_row_id": np.array([0, 1, 0, 0, 1], dtype=np.int32), + "wages": [52000.0, 0.0, 18750.5, 9100.0, 0.0], + } + ) + tables = {"person": person} + for group in groups: + ids = [1, 2, 3] if group == "household" else [11, 21, 31, 32] + tables[group] = pd.DataFrame({US_SHAPED_SCHEMA.id_column(group): ids}) + return Frame( + tables, + US_SHAPED_SCHEMA, + { + "household": Weights(US_SHAPED_HOUSEHOLD_WEIGHTS, WeightKind.IMPORTANCE), + "person": Weights(US_SHAPED_PERSON_WEIGHTS, WeightKind.DESIGN), + }, + pd.Series(["asec_2023", "asec_2023", "asec_2024", "asec_2024", "asec_2024"]), + ) + + +def _us_shaped_mint(ids: np.ndarray, k: int) -> np.ndarray: + return ids + k * US_SHAPED_MULTIPLIER + + +def _us_shaped_require_below(ids: np.ndarray, what: str) -> None: + if (np.asarray(ids) >= US_SHAPED_MULTIPLIER).any(): + raise ValueError(f"Some {what} reach the multiplier.") + + +US_SHAPED_DECLARATION = CloneDeclaration( + ids=CloneIdRule( + bound=len(CHANNELS), + mint=_us_shaped_mint, + require_below=_us_shaped_require_below, + ), + clone_index_columns={ + entity: f"{entity}_support_clone_index" for entity in US_SHAPED_SCHEMA.entities + }, + weight_kinds=frozenset({WeightKind.DESIGN, WeightKind.IMPORTANCE}), + reason="Support clones split each household's weight equally.", + source_id_columns={ + entity: f"{entity}_source_id" for entity in US_SHAPED_SCHEMA.entities + }, +) + + +def test_us_shaped_clones_pass_the_shared_clone_invariant() -> None: + frame = _us_shaped_frame() + expanded = expand_clones( + frame, clones=len(CHANNELS), declaration=US_SHAPED_DECLARATION + ) + labels = np.asarray(CHANNELS, dtype=object) + tables = {} + for entity in expanded.entities: + table = expanded.table(entity).copy() + index = table[f"{entity}_support_clone_index"].to_numpy() + table[f"{entity}_support_channel"] = labels[index] + tables[entity] = table + labelled = Frame( + tables, + expanded.schema, + {entity: expanded.weights_for(entity) for entity in expanded.weighted_entities}, + expanded.strata, + mass_log=expanded.mass_log, + ) + assert_clone_expansion(frame, labelled, channels=CHANNELS) + # Both weighted entities split, each keeping its kind, with a record each. + for entity, values, kind in ( + ("person", US_SHAPED_PERSON_WEIGHTS, WeightKind.DESIGN), + ("household", US_SHAPED_HOUSEHOLD_WEIGHTS, WeightKind.IMPORTANCE), + ): + weights = labelled.weights_for(entity) + assert weights.kind is kind + assert weights.values.tobytes() == np.tile(values / 2, 2).tobytes() + assert [record.entity for record in labelled.mass_log] == ["person", "household"] + + +# --------------------------------------------------------------------------- +# Refusals: each names entities, columns, kinds and declared constants only. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("clones", [0, -1, BOUND + 1, True, 2.0, "2", np.int64(0)]) +def test_a_clone_count_outside_the_declared_bound_is_refused(clones: object) -> None: + frame = _frame() + with pytest.raises( + ValueError, match=r"an integer in \[1, 5\], the declared bound" + ) as error: + expand_clones(frame, clones=clones, declaration=DECLARATION) # type: ignore[arg-type] + _assert_names_no_value(error, frame) + + +def test_a_frame_that_already_carries_a_declared_column_is_refused() -> None: + once = expand_clones(_frame(), clones=2, declaration=DECLARATION) + with pytest.raises(ValueError, match="already carries the declared") as error: + expand_clones(once, clones=2, declaration=DECLARATION) + assert str(error.value).endswith(f"{sorted(DECLARED_COLUMNS)}.") + _assert_names_no_value(error, once) + tables = _tables() + tables["household"]["household_origin"] = tables["household"]["household_id"] + frame = _frame(tables) + with pytest.raises( + ValueError, match=r"column\(s\) \['household_origin'\]" + ) as error: + expand_clones(frame, clones=2, declaration=DECLARATION) + _assert_names_no_value(error, frame) + + +def test_ids_the_mint_has_no_room_for_are_refused_by_the_declared_rule() -> None: + frame = _frame(_tables(households=(411, 522, RADIX + 633))) + with pytest.raises(ValueError, match="1 of the household_id values reach") as error: + expand_clones(frame, clones=2, declaration=DECLARATION) + _assert_names_no_value(error, frame) + handed: list[tuple[str, int]] = [] + + def record(ids: np.ndarray, what: str) -> None: + handed.append((what, len(ids))) + + rule = CloneIdRule(bound=BOUND, mint=_mint, require_below=record) + expand_clones(_frame(), clones=2, declaration=_declaration(ids=rule)) + assert handed == [ + ("person_id values", 6), + ("unit_id values", 4), + ("household_id values", 3), + ] + + +def test_a_weight_kind_the_declaration_does_not_accept_is_refused() -> None: + frame = _frame(kind=WeightKind.IMPORTANCE) + with pytest.raises( + ValueError, + match=r"accepts \['design'\] weights, not 'importance' weights on 'household'", + ) as error: + expand_clones(frame, clones=2, declaration=DECLARATION) + _assert_names_no_value(error, frame) + # Every weighted entity's kind is checked, not the household's alone. + frame = _frame( + extra_weights={"person": Weights(np.full(6, 2.5), WeightKind.IMPORTANCE)} + ) + with pytest.raises( + ValueError, match="not 'importance' weights on 'person'" + ) as error: + expand_clones(frame, clones=2, declaration=DECLARATION) + _assert_names_no_value(error, frame) + + +def _offset_mint(ids: np.ndarray, k: int) -> np.ndarray: + return ids + k * 10 + + +def _reversing_mint(ids: np.ndarray, k: int) -> np.ndarray: + return k * RADIX + (RADIX - 1 - ids) + + +def _float_mint(ids: np.ndarray, k: int) -> np.ndarray: + return (ids + k * RADIX).astype(np.float64) + + +def _short_mint(ids: np.ndarray, k: int) -> np.ndarray: + return (ids + k * RADIX)[:-1] + + +def _uint64_mint(ids: np.ndarray, k: int) -> np.ndarray: + return (ids + k * RADIX).astype(np.uint64) + + +def _int64_mint(ids: np.ndarray, k: int) -> np.ndarray: + return ids.astype(np.int64) + k * RADIX + + +_OWN_DTYPE = "one integer for each person id, in the ids' own dtype" + + +@pytest.mark.parametrize( + ("mint", "match"), + [ + (_offset_mint, "person ids above the clone before's"), + (_reversing_mint, "keep the order of the person ids"), + (_float_mint, _OWN_DTYPE), + (_short_mint, _OWN_DTYPE), + (_uint64_mint, _OWN_DTYPE), + ], +) +def test_a_mint_that_breaks_the_clone_id_rule_is_refused(mint, match: str) -> None: + frame = _frame() + rule = CloneIdRule(bound=BOUND, mint=mint, require_below=_require_below) + with pytest.raises(ValueError, match=match) as error: + expand_clones(frame, clones=3, declaration=_declaration(ids=rule)) + _assert_names_no_value(error, frame) + + +def _with_id_dtype(dtype: str, *, memberships: str | None = None) -> dict: + """The invented tables with every id in ``dtype``, and every membership + in ``memberships`` (``dtype`` by default).""" + + tables = _tables() + for entity, table in tables.items(): + column = SCHEMA.entity_id_column(entity) + table[column] = table[column].astype(dtype) + for group in SCHEMA.group_entities: + column = SCHEMA.membership_column(group) + tables["person"][column] = tables["person"][column].astype(memberships or dtype) + return tables + + +@pytest.mark.parametrize("clones", [1, 3]) +def test_a_mint_in_the_ids_dtype_keeps_it_at_every_clone_count(clones: int) -> None: + frame = _frame(_with_id_dtype("int32")) + expanded = expand_clones(frame, clones=clones, declaration=DECLARATION) + for entity in frame.entities: + ids = expanded.table(entity)[SCHEMA.entity_id_column(entity)] + assert ids.dtype == np.int32, entity + assert ids.tolist() == [ + value + k * RADIX + for k in range(clones) + for value in frame.table(entity)[ids.name].tolist() + ] + person = expanded.table("person") + for group in SCHEMA.group_entities: + assert person[SCHEMA.membership_column(group)].dtype == np.int32, group + assert expanded.table("household")[SOURCE_COLUMNS["household"]].dtype == np.int32 + + +def test_a_mint_out_of_the_ids_dtype_is_refused_before_it_floats_them() -> None: + # Unsigned ids with a mint that returns int64, as a mint that casts to + # int64 does: concatenating the two would give float64 ids at K = 2. + frame = _frame(_with_id_dtype("uint64")) + rule = CloneIdRule(bound=BOUND, mint=_int64_mint, require_below=_require_below) + once = expand_clones(frame, clones=1, declaration=_declaration(ids=rule)) + assert once.table("household")["household_id"].dtype == np.uint64 + with pytest.raises(ValueError, match=_OWN_DTYPE) as error: + expand_clones(frame, clones=2, declaration=_declaration(ids=rule)) + _assert_names_no_value(error, frame) + + +@pytest.mark.parametrize( + ("ids", "memberships"), [("int64", "int32"), ("uint64", "int64")] +) +def test_memberships_in_another_dtype_than_their_units_ids_are_refused( + ids: str, memberships: str +) -> None: + # Concatenating the members with their units' minted ids would change + # the memberships' dtype at K = 2, or float them. + frame = _frame(_with_id_dtype(ids, memberships=memberships)) + with pytest.raises( + TypeError, + match=f"'person_unit_id' in the dtype of 'unit_id'; it holds {memberships} " + f"against {ids}", + ) as error: + expand_clones(frame, clones=2, declaration=DECLARATION) + _assert_names_no_value(error, frame) + + +def test_ids_that_are_not_integers_are_refused() -> None: + tables = _tables() + tables["household"]["household_id"] = tables["household"]["household_id"].astype( + np.float64 + ) + tables["person"]["person_household_id"] = tables["person"][ + "person_household_id" + ].astype(np.float64) + frame = _frame(tables) + with pytest.raises(TypeError, match="'household_id' holds float64") as error: + expand_clones(frame, clones=2, declaration=DECLARATION) + _assert_names_no_value(error, frame) + + +def test_a_frame_with_a_link_table_is_refused() -> None: + schema = EntitySchema( + group_entities=SCHEMA.group_entities, + links=(LinkSpec("unit_household", "unit", "household"),), + ) + tables = _tables() + tables["unit_household"] = pd.DataFrame( + {"unit_id": [4111, 4112, 5221, 6331], "household_id": [411, 411, 522, 633]} + ) + frame = _frame(tables, schema=schema) + with pytest.raises(ValueError, match="copies no link table") as error: + expand_clones(frame, clones=2, declaration=DECLARATION) + _assert_names_no_value(error, frame) + + +def test_a_declaration_that_does_not_fit_the_frame_s_entities_is_refused() -> None: + frame = _frame() + lacking = _declaration( + clone_index_columns={**INDEX_COLUMNS, "family": "family_clone"} + ) + with pytest.raises(ValueError, match=r"entities the frame lacks: \['family'\]"): + expand_clones(frame, clones=2, declaration=lacking) + partial = _declaration( + clone_index_columns={ + entity: column + for entity, column in INDEX_COLUMNS.items() + if entity != "unit" + } + ) + with pytest.raises(ValueError, match=r"gives \['unit'\] no clone-index column"): + expand_clones(frame, clones=2, declaration=partial) + + +def test_clones_whose_total_weight_strays_are_refused(monkeypatch) -> None: + def biased(values: np.ndarray, count: int) -> np.ndarray: + return np.tile(values / count, count) * (1.0 + 1e-6) + + monkeypatch.setattr(clone_expansion, "_equal_shares", biased) + frame = _frame() + with pytest.raises( + ValueError, match="beyond the relative tolerance 1e-09" + ) as error: + expand_clones(frame, clones=3, declaration=DECLARATION) + _assert_names_no_value(error, frame) + + +@pytest.mark.parametrize( + ("make", "error"), + [ + ( + lambda: CloneIdRule(bound=0, mint=_mint, require_below=_require_below), + ValueError, + ), + ( + lambda: CloneIdRule(bound=True, mint=_mint, require_below=_require_below), + ValueError, + ), + ( + lambda: CloneIdRule(bound=2.0, mint=_mint, require_below=_require_below), + ValueError, + ), + ( + lambda: CloneIdRule(bound=2, mint=None, require_below=_require_below), + TypeError, + ), + (lambda: _declaration(ids=_mint), TypeError), + (lambda: _declaration(clone_index_columns={}), ValueError), + (lambda: _declaration(clone_index_columns=["person_clone"]), TypeError), + (lambda: _declaration(clone_index_columns={"person": ""}), ValueError), + ( + lambda: _declaration(source_id_columns={"household": "household_clone"}), + ValueError, + ), + (lambda: _declaration(weight_kinds=frozenset()), ValueError), + (lambda: _declaration(weight_kinds=frozenset({"design"})), ValueError), + (lambda: _declaration(reason=" "), ValueError), + ], +) +def test_a_malformed_declaration_is_refused(make, error: type[Exception]) -> None: + with pytest.raises(error): + make() + + +def test_a_declaration_s_columns_cannot_be_changed_after_it_is_made() -> None: + columns = dict(INDEX_COLUMNS) + declaration = _declaration(clone_index_columns=columns) + columns["person"] = "another" + assert declaration.clone_index_columns["person"] == "person_clone" + with pytest.raises(TypeError): + declaration.clone_index_columns["person"] = "another" # type: ignore[index] + + +def test_a_declaration_hashes_by_identity() -> None: + # Its mappings are frozen views, which do not hash, so a declaration + # compares and hashes as itself and can key a cache. + assert {DECLARATION: "cached"}[DECLARATION] == "cached" + assert _declaration() != DECLARATION diff --git a/packages/microcosm-build/tests/engine_free/shared/test_source_lineage.py b/packages/microcosm-build/tests/engine_free/shared/test_source_lineage.py index 1656bc9bc5..01d24cabf6 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_source_lineage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_source_lineage.py @@ -2,6 +2,9 @@ from __future__ import annotations +import math +from itertools import permutations + import numpy as np import pandas as pd import pytest @@ -9,8 +12,10 @@ from microcosm.build.source_lineage import ( CopyBranch, CopyLineage, + ReplicaFold, collapse_replicas, copy_paths, + fold_replicas, record_sources, replica_keys, ) @@ -316,3 +321,318 @@ def test_inputs_must_align_and_name_each_column_once() -> None: duplicated = pd.DataFrame([[1.0, 2.0], [1.0, 2.0]], columns=["x", "x"]) with pytest.raises(ValueError, match="each column once"): collapse_replicas(keys, rows, weights, inputs=duplicated) + + +# --- The fold's representatives, variety and key groups (microcosm#931) ---- + + +def _fold_by_hand( + keys: np.ndarray, + rows: np.ndarray, + weights: np.ndarray, + inputs: pd.DataFrame | None, + order: np.ndarray | None, +) -> dict[str, np.ndarray]: + """The fold's contract, record by record in plain Python. + + A variant is a key with one set of input values, a missing value + matching only a missing value. Its representative is its selected + record with the least order, then the earliest row (the earliest row + alone without an order), and carries the variant's weights summed + exactly. A selected record varies when its key holds two variants or + more, and its key group numbers its key by first appearance. + """ + + def cell(value: object) -> object: + return None if isinstance(value, float) and math.isnan(value) else value + + def variant(row: int) -> tuple[object, ...]: + values = () if inputs is None else tuple(map(cell, inputs.iloc[row])) + return (keys[row], *values) + + def rank(row: int) -> tuple[int, int]: + return (0 if order is None else int(order[row]), row) + + selected = np.flatnonzero(rows).tolist() + members: dict[tuple[object, ...], list[int]] = {} + codes: dict[object, int] = {} + for row in selected: + members.setdefault(variant(row), []).append(row) + codes.setdefault(keys[row], len(codes)) + variants = {key: 0 for key in codes} + for key, *_ in members: + variants[key] += 1 + kept = np.zeros(len(keys), dtype=bool) + summed = np.asarray(weights, dtype=np.float64).copy() + representative = np.full(len(keys), -1, dtype=np.int64) + for group in members.values(): + first = min(group, key=rank) + kept[first] = True + summed[first] = math.fsum(float(weights[row]) for row in group) + representative[group] = first + varied = np.zeros(len(keys), dtype=bool) + key_group = np.full(len(keys), -1, dtype=np.int64) + for row in selected: + varied[row] = variants[keys[row]] > 1 + key_group[row] = codes[keys[row]] + return { + "kept": kept, + "weights": summed, + "representative": representative, + "varied": varied, + "key_group": key_group, + } + + +_FOLD_FIELDS = ("kept", "weights", "representative", "varied", "key_group") + + +def _assert_fold(fold: ReplicaFold, expected: dict[str, np.ndarray]) -> None: + """Each named field of ``fold`` equals ``expected``'s, dtype and bytes.""" + + for name, values in expected.items(): + found = getattr(fold, name) + assert found.dtype == values.dtype, name + assert found.tobytes() == values.tobytes(), name + + +def test_each_record_trains_on_its_least_order_clone_in_any_row_order() -> None: + """Two records placed three times, clone k at id + 1000 k: whatever the + rows' order, each record trains on clone 0 with all three clones' + weight, and every clone points at it.""" + + keys = np.asarray(["r1", "r2"] * 3, dtype=object) + ids = np.asarray([11, 12, 1011, 1012, 2011, 2012], dtype=np.int64) + weights = np.asarray([0.1, 0.25, 0.2, 0.25, 0.7, 0.5]) + agreeing = pd.DataFrame({"value": np.zeros(6)}) + clone_zero = {"r1": 11, "r2": 12} + for permutation in permutations(range(6)): + shuffled = np.asarray(permutation) + for inputs in (None, agreeing): + fold = fold_replicas( + keys[shuffled], + np.ones(6, dtype=bool), + weights[shuffled], + inputs=inputs, + order=ids[shuffled], + ) + assert sorted(ids[shuffled][fold.kept].tolist()) == [11, 12] + # 0.1 + 0.2 + 0.7 rounds once to 1.0, as does 0.25 + 0.25 + 0.5. + assert fold.weights[fold.kept].tolist() == [1.0, 1.0] + assert ids[shuffled][fold.representative].tolist() == [ + clone_zero[key] for key in keys[shuffled] + ] + assert not fold.varied.any() + + +def test_each_variant_trains_on_its_own_least_order_replica() -> None: + keys = np.asarray(["a", "a", "a", "a", "b", "b"], dtype=object) + rows = np.ones(6, dtype=bool) + weights = np.asarray([0.5, 0.25, 0.125, 1.0, 2.0, 4.0]) + inputs = pd.DataFrame({"band": ["x", "y", "x", "y", "z", "z"]}) + order = np.asarray([30, 10, 20, 40, 50, 5], dtype=np.int64) + fold = fold_replicas(keys, rows, weights, inputs=inputs, order=order) + # a's variants x (rows 0, 2) and y (rows 1, 3) each keep their own + # least order; b's one variant keeps its later row, which orders first. + assert fold.kept.tolist() == [False, True, True, False, False, True] + assert fold.representative.tolist() == [2, 1, 2, 1, 5, 5] + assert fold.weights.tolist() == [0.5, 1.25, 0.625, 1.0, 2.0, 6.0] + assert fold.varied.tolist() == [True, True, True, True, False, False] + assert fold.key_group.tolist() == [0, 0, 0, 0, 1, 1] + # Replicas tied on the order go to the earlier row. + tied = fold_replicas( + keys, rows, weights, inputs=inputs, order=np.zeros(6, dtype=np.int64) + ) + assert tied.representative.tolist() == [0, 1, 0, 1, 4, 4] + + +def test_a_fold_without_replicas_is_the_identity() -> None: + keys = np.asarray(["a", "b", "a"], dtype=object) + rows = np.asarray([True, True, False]) + weights = np.asarray([0.1, 0.2, 0.3]) + for inputs in (None, pd.DataFrame({"value": [1.0, 2.0, 3.0]})): + for order in (None, np.asarray([9, 8, 7], dtype=np.int64)): + fold = fold_replicas(keys, rows, weights, inputs=inputs, order=order) + _assert_fold( + fold, + { + "kept": rows, + "weights": weights, + "representative": np.asarray([0, 1, -1], dtype=np.int64), + "varied": np.zeros(3, dtype=bool), + "key_group": np.asarray([0, 1, -1], dtype=np.int64), + }, + ) + + +def test_the_fold_matches_hand_built_maps() -> None: + keys = np.asarray(["a", "a", "a", "a", "b", "b", "c", "d"], dtype=object) + rows = np.asarray([True] * 7 + [False]) + weights = np.asarray([0.5, 0.25, 0.125, 1.0, 2.0, 4.0, 8.0, 9.0]) + inputs = pd.DataFrame({"income": [3.0, 0.0, 3.0, np.nan, 7.0, 7.0, 1.0, 2.0]}) + order = np.asarray([40, 30, 20, 10, 60, 50, 70, 80], dtype=np.int64) + fold = fold_replicas(keys, rows, weights, inputs=inputs, order=order) + # a holds three variants (3.0 on rows 0 and 2, 0.0, missing); b agrees + # and folds onto its lower order; c is alone; d is not selected. + _assert_fold( + fold, + { + "kept": np.asarray([False, True, True, True, False, True, True, False]), + "weights": np.asarray([0.5, 0.25, 0.625, 1.0, 2.0, 6.0, 8.0, 9.0]), + "representative": np.asarray([2, 1, 2, 3, 5, 5, 6, -1], dtype=np.int64), + "varied": np.asarray([True] * 4 + [False] * 4), + "key_group": np.asarray([0, 0, 0, 0, 1, 1, 2, -1], dtype=np.int64), + }, + ) + _assert_fold(fold, _fold_by_hand(keys, rows, weights, inputs, order)) + kept, summed = collapse_replicas(keys, rows, weights, inputs=inputs, order=order) + assert kept.tobytes() == fold.kept.tobytes() + assert summed.tobytes() == fold.weights.tobytes() + + +_TODAYS_FIXTURE_INPUTS = pd.DataFrame( + { + "income": [3.0, 0.0, 3.0, np.nan, 7.0, 7.0], + "band": pd.Categorical(["x", "x", "x", "x", "y", "y"]), + } +) + + +@pytest.mark.parametrize( + ("keys", "rows", "weights", "inputs", "kept", "summed", "representative"), + [ + ( + ["a", "b", "a", "c", "b", "a"], + [True, True, True, False, True, True], + [0.1, 0.2, 0.2, 9.0, 0.3, 0.7], + None, + [True, True, False, False, False, False], + [1.0, 0.5, 0.2, 9.0, 0.3, 0.7], + [0, 1, 0, -1, 1, 0], + ), + ( + ["a", "b", "a", "c", "b", "a"], + [True, True, True, False, True, True], + [0.1, 0.2, 0.2, 9.0, 0.3, 0.7], + pd.DataFrame({"value": np.ones(6)}), + [True, True, False, False, False, False], + [1.0, 0.5, 0.2, 9.0, 0.3, 0.7], + [0, 1, 0, -1, 1, 0], + ), + ( + ["a", "a", "a", "a", "b", "b"], + [True] * 6, + [0.5, 0.25, 0.25, 1.0, 2.0, 2.0], + _TODAYS_FIXTURE_INPUTS, + [True, True, False, True, True, False], + [0.75, 0.25, 0.25, 1.0, 4.0, 2.0], + [0, 1, 0, 3, 4, 4], + ), + ], + ids=["keys_alone", "agreeing_inputs", "variants"], +) +def test_without_an_order_each_variant_keeps_its_first_row( + keys, rows, weights, inputs, kept, summed, representative +) -> None: + """``order=None`` is the fold as it was, on the keys and inputs of the + fixtures above: each variant keeps its first row, as the row positions + taken as the order give too.""" + + keys = np.asarray(keys, dtype=object) + rows, weights = np.asarray(rows), np.asarray(weights) + fold = fold_replicas(keys, rows, weights, inputs=inputs) + assert fold.kept.tolist() == kept + assert fold.weights.tolist() == summed + assert fold.representative.tolist() == representative + by_position = fold_replicas( + keys, rows, weights, inputs=inputs, order=np.arange(len(keys)) + ) + _assert_fold(fold, {name: getattr(by_position, name) for name in _FOLD_FIELDS}) + collapsed = collapse_replicas(keys, rows, weights, inputs=inputs) + assert collapsed[0].tobytes() == fold.kept.tobytes() + assert collapsed[1].tobytes() == fold.weights.tobytes() + + +def test_clones_that_all_differ_fold_nothing_yet_vary() -> None: + """Two clones of each record differ on a continuous input: nothing + folds, as before, and every record varies. A fold that decided its + early return on the key and the inputs together would report no + variety exactly here.""" + + keys = np.asarray(["a", "b", "a", "b"], dtype=object) + rows = np.ones(4, dtype=bool) + weights = np.asarray([0.5, 0.25, 0.5, 0.25]) + inputs = pd.DataFrame({"income": [1.5, 2.5, 3.5, 4.5]}) + order = np.asarray([1, 2, 1001, 1002], dtype=np.int64) + fold = fold_replicas(keys, rows, weights, inputs=inputs, order=order) + _assert_fold( + fold, + { + "kept": rows, + "weights": weights, + "representative": np.arange(4, dtype=np.int64), + "varied": np.ones(4, dtype=bool), + "key_group": np.asarray([0, 1, 0, 1], dtype=np.int64), + }, + ) + kept, summed = collapse_replicas(keys, rows, weights, inputs=inputs) + assert kept.tolist() == rows.tolist() + assert summed.tobytes() == weights.tobytes() + + +def test_the_fold_keeps_its_contract_on_random_frames() -> None: + rng = np.random.default_rng(17) + letters = np.asarray(["a", "b", "c", "d"], dtype=object) + for trial in range(240): + n = int(rng.integers(0, 24)) + keys = rng.choice(letters, n) + rows = rng.random(n) < 0.8 + weights = rng.uniform(0.1, 3.0, n) + inputs = None + if trial % 3: + inputs = pd.DataFrame( + { + "value": rng.choice([0.0, 1.0, np.nan], n), + "band": rng.choice(np.asarray(["x", "y"], dtype=object), n), + } + ) + # A narrow range of orders makes ties, which go to the earlier row. + order = None if trial % 2 else rng.integers(0, 4, n) + fold = fold_replicas(keys, rows, weights, inputs=inputs, order=order) + _assert_fold(fold, _fold_by_hand(keys, rows, weights, inputs, order)) + + +@pytest.mark.parametrize( + "order", + [ + np.asarray([7357.5, 7358.5, 7359.5]), + np.asarray([7357, 7358], dtype=np.int64), + np.asarray([[7357, 7358, 7359]], dtype=np.int64), + np.asarray([True, False, True]), + np.asarray(["7357", "7358", "7359"], dtype=object), + np.asarray([7357, 7358, 7359], dtype="m8[ns]"), + ], + ids=["float", "short", "two_dimensional", "boolean", "text", "timedelta"], +) +def test_the_order_is_one_integer_per_record_refused_by_count(order) -> None: + for keys in (["a", "b", "c"], ["a", "a", "b"]): + with pytest.raises(ValueError, match="one integer per record") as refusal: + fold_replicas( + np.asarray(keys, dtype=object), + np.ones(3, dtype=bool), + np.ones(3), + order=order, + ) + # The refusal counts values and records; it prints no order value. + assert "735" not in str(refusal.value) + + +def test_a_selected_record_without_a_key_is_refused() -> None: + # A null key names no record, so its record would fold with others. + keys = np.asarray(["a", None, "a", None], dtype=object) + with pytest.raises(ValueError, match="must not be null"): + fold_replicas(keys, np.ones(4, dtype=bool), np.ones(4)) + # Off the selection a key is never read. + fold = fold_replicas(keys, np.asarray([True, False, True, False]), np.ones(4)) + assert fold.kept.tolist() == [True, False, False, False] + assert fold.key_group.tolist() == [0, -1, 0, -1] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py index 55c84e7b26..7ecb814bcb 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py @@ -217,6 +217,41 @@ def test_a_repeated_id_is_refused() -> None: integer_record_keys(np.array([1, 2, 1]), label="ids") +def test_integer_record_keys_find_repeats_as_np_unique_does() -> None: + # The sort-based check gives np.unique's verdict on every array: random + # integer ids of several dtypes, narrow ranges making repeats likely, + # empty and single-id arrays, and the extremes of int64 (microcosm#931). + # Ids that are not one-dimensional are still refused by type first, and + # so are timedeltas, numpy integers by subtype whose NaT never repeat. + for refused in ( + np.array([[1, 2], [1, 2]]), + np.array([1, 2], dtype="m8[ns]"), + np.array(["NaT", "NaT"], dtype="m8[ns]"), + ): + with pytest.raises(TypeError, match="one-dimensional"): + integer_record_keys(refused, label="ids") + rng = np.random.default_rng(23) + dtypes = (np.int64, np.int32, np.int8, np.uint64, np.uint16) + verdicts = {True: 0, False: 0} + extremes = np.array([np.iinfo(np.int64).min, -1, 0, np.iinfo(np.int64).max]) + arrays = [extremes, np.append(extremes, extremes[-1]), np.array([], dtype="int64")] + for trial in range(400): + n = int(rng.integers(0, 50)) + span = int(rng.integers(1, 120)) + arrays.append(rng.integers(0, span, n).astype(dtypes[trial % len(dtypes)])) + for ids in arrays: + repeated = np.unique(ids).size != ids.size + verdicts[repeated] += 1 + if repeated: + with pytest.raises(ValueError, match="each record once"): + integer_record_keys(ids, label="ids") + else: + keys = integer_record_keys(ids, label="ids") + assert keys.dtype == ids.dtype + np.testing.assert_array_equal(keys, ids) + assert min(verdicts.values()) > 50 + + def test_balanced_uniforms_give_each_stratum_its_rate_to_within_half_a_record() -> None: # At midpoints the last record taken sits below the rate and the next one # at or above it, so a stratum misses by at most half its heaviest record diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_expansion_core.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_expansion_core.py new file mode 100644 index 0000000000..818c1bf883 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_expansion_core.py @@ -0,0 +1,249 @@ +"""The shared clone expansion core under the UK's declarations (microcosm#931). + +With the copy-id table's ``geographic_support`` field, the ladder's +clone-index columns and household design weights, the core must give what the +clone-first stand-in gives (``clone_first_frame`` in +test_support/microcosm_build/uk_clone_first_frames.py, written without the +core): bit for bit on the five id columns, the household clone index, the +source household id and the weights, at K = 1, 2, 3, 7 and 15. Invented +frames only; the refusals print counts and declared constants, never an id. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.clone_expansion import ( + CloneDeclaration, + CloneIdRule, + expand_clones, +) +from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN +from test_support.microcosm_build.uk_clone_first_frames import ( + clone_first_frame, + with_last_household_moved, +) + +_FIELD = "geographic_support" +_REASON = "Geographic clones split each household's weight equally; mass is conserved." + + +def _mint_geographic_clone(ids: np.ndarray, k: int) -> np.ndarray: + return copy_id_table.mint(ids, _FIELD, k) + + +def _require_room_for_geographic_clones(ids: np.ndarray, what: str) -> None: + copy_id_table.require_below(ids, _FIELD, what=what) + + +UK_GEOGRAPHIC_CLONES = CloneDeclaration( + ids=CloneIdRule( + bound=copy_id_table.ordinal_bound(_FIELD), + mint=_mint_geographic_clone, + require_below=_require_room_for_geographic_clones, + ), + clone_index_columns={ + entity: ladder_clone_index_column(entity) + for entity in UK_NATIONAL_SCHEMA.entities + }, + weight_kinds=frozenset({WeightKind.DESIGN}), + reason=_REASON, + source_id_columns={"household": SOURCE_HOUSEHOLD_ID_COLUMN}, +) +_ID_COLUMNS = { + "person": ("person_id", "person_household_id", "person_benunit_id"), + "benunit": ("benunit_id",), + "household": ("household_id",), +} +#: The stand-in's columns the core must reproduce bit for bit. +_ORACLE_COLUMNS = { + **_ID_COLUMNS, + "household": ( + "household_id", + ladder_clone_index_column("household"), + SOURCE_HOUSEHOLD_ID_COLUMN, + ), +} +_CLONE_COUNTS = (1, 2, 3, 7, 15) + + +def _uk_frame(households: int = 12) -> Frame: + """Invented FRS-shaped ids: household sernums, benefit unit sernum x 100 + + b, person sernum x 1000 + p; one to four people in one or two units.""" + + people: list[tuple[int, int, int, float]] = [] + units: list[int] = [] + for sernum in range(1, households + 1): + size = 1 + sernum % 4 + unit_count = min(size, 1 + sernum % 2) + units.extend(sernum * 100 + b for b in range(1, unit_count + 1)) + people.extend( + ( + sernum * 1000 + p, + sernum, + sernum * 100 + min(p, unit_count), + 875.5 * p + sernum, + ) + for p in range(1, size + 1) + ) + person = pd.DataFrame( + people, + columns=[ + "person_id", + "person_household_id", + "person_benunit_id", + "employment_income", + ], + ).astype( + { + "person_id": "int64", + "person_household_id": "int64", + "person_benunit_id": "int64", + } + ) + sernums = range(1, households + 1) + household = pd.DataFrame( + { + "household_id": np.arange(1, households + 1, dtype=np.int64), + "region": ["WALES" if s % 3 == 0 else "NORTH_EAST" for s in sernums], + "household_label": pd.Series([f"2024:{s}" for s in sernums], dtype=object), + } + ) + return uk_national_frame( + person=person, + benunit=pd.DataFrame({"benunit_id": np.asarray(units, dtype=np.int64)}), + household=household, + household_weights=np.random.default_rng(931).uniform(400.0, 3000.0, households), + time_period="2024", + ) + + +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_the_core_under_uk_declarations_is_the_clone_first_stand_in( + clones: int, +) -> None: + frame = _uk_frame() + expanded = expand_clones(frame, clones=clones, declaration=UK_GEOGRAPHIC_CLONES) + stand_in = clone_first_frame(frame, clones) + for entity, columns in _ORACLE_COLUMNS.items(): + for column in columns: + got = expanded.table(entity)[column].to_numpy() + want = stand_in.table(entity)[column].to_numpy() + assert got.dtype == want.dtype, (entity, column) + assert got.tobytes() == want.tobytes(), (entity, column) + got_weights = expanded.weights_for("household") + want_weights = stand_in.weights_for("household") + assert got_weights.kind is want_weights.kind is WeightKind.DESIGN + assert got_weights.values.tobytes() == want_weights.values.tobytes() + # Every column the stand-in carries, in its order and dtypes. + for entity in stand_in.entities: + table = stand_in.table(entity) + pd.testing.assert_frame_equal( + expanded.table(entity)[list(table.columns)], table, check_exact=True + ) + assert uk_time_period(expanded) == uk_time_period(stand_in) + + +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_every_id_decodes_to_its_clone_and_its_clone_0_row(clones: int) -> None: + frame = _uk_frame() + expanded = expand_clones(frame, clones=clones, declaration=UK_GEOGRAPHIC_CLONES) + validate_uk_national_frame(expanded) + for entity, columns in _ID_COLUMNS.items(): + table = expanded.table(entity) + index = table[ladder_clone_index_column(entity)] + assert index.dtype == np.int64 + assert sorted(set(index.tolist())) == list(range(clones)) + for column in columns: + ids = table[column].to_numpy() + # The field's digits are the row's clone, and clearing them gives + # the clone-0 row's id: the lineage the expand kernel recovers. + assert np.array_equal(copy_id_table.ordinal(ids, _FIELD), index.to_numpy()) + assert np.array_equal( + copy_id_table.parent(ids, _FIELD), + np.tile(frame.table(entity)[column].to_numpy(), clones), + ) + # Every member carries its household's and its benefit unit's clone. + person = expanded.table("person") + for group in UK_NATIONAL_SCHEMA.group_entities: + units = expanded.table(group).set_index(UK_NATIONAL_SCHEMA.id_column(group)) + assert np.array_equal( + units.loc[ + person[UK_NATIONAL_SCHEMA.membership_column(group)], + ladder_clone_index_column(group), + ], + person[ladder_clone_index_column("person")], + ) + + +def test_the_mass_record_is_the_same_at_every_clone_count() -> None: + frame = _uk_frame() + records = [ + expand_clones(frame, clones=clones, declaration=UK_GEOGRAPHIC_CLONES) + for clones in _CLONE_COUNTS + ] + assert { + (expanded.mass_log[-1].entity, expanded.mass_log[-1].declared_factor) + for expanded in records + } == {("household", 1.0)} + assert {expanded.mass_log[-1].reason for expanded in records} == {_REASON} + + +def test_a_python_string_column_stays_python_strings_in_every_clone() -> None: + frame = _uk_frame() + expanded = expand_clones(frame, clones=3, declaration=UK_GEOGRAPHIC_CLONES) + labels = expanded.table("household")["household_label"] + assert labels.dtype == object + assert {type(value) for value in labels.tolist()} == {str} + assert labels.tolist() == frame.table("household")["household_label"].tolist() * 3 + + +@pytest.mark.parametrize("clones", [0, copy_id_table.ordinal_bound(_FIELD) + 1]) +def test_a_clone_count_outside_the_field_s_bound_is_refused(clones: int) -> None: + with pytest.raises( + ValueError, match=r"an integer in \[1, 100\], the declared bound" + ): + expand_clones(_uk_frame(), clones=clones, declaration=UK_GEOGRAPHIC_CLONES) + + +def test_an_id_in_the_field_is_refused_by_count() -> None: + radix = copy_id_table.radix(_FIELD) + moved = with_last_household_moved(_uk_frame(), radix) + with pytest.raises( + ValueError, + match="1 of the household_id values reach the 'geographic_support' copy-id", + ) as error: + expand_clones(moved, clones=2, declaration=UK_GEOGRAPHIC_CLONES) + moved_id = int(moved.table("household")["household_id"].iloc[-1]) + assert str(moved_id) not in str(error.value) + + +def test_importance_weights_and_a_second_expansion_are_refused() -> None: + frame = _uk_frame() + importance = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + { + "household": Weights( + frame.weights_for("household").values, WeightKind.IMPORTANCE + ) + }, + frame.strata, + metadata=frame.metadata, + ) + with pytest.raises(ValueError, match=r"accepts \['design'\] weights"): + expand_clones(importance, clones=2, declaration=UK_GEOGRAPHIC_CLONES) + once = expand_clones(frame, clones=2, declaration=UK_GEOGRAPHIC_CLONES) + with pytest.raises(ValueError, match="already carries the declared column"): + expand_clones(once, clones=2, declaration=UK_GEOGRAPHIC_CLONES) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_clone_collapse.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_clone_collapse.py index 257075a5fc..b661f85323 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_clone_collapse.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_clone_collapse.py @@ -14,8 +14,12 @@ from microcosm.build.uk_runtime.household_lineage import ( UK_COPY_LINEAGE, collapse_geographic_clones, + fold_geographic_clones, +) +from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + uk_national_frame, ) -from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column from test_support.microcosm_build.uk_geographic_clones import with_geographic_clones @@ -87,7 +91,8 @@ def test_only_geographic_clones_fold_into_their_survey_record() -> None: inputs=_agreeing(n), ) clone = pool.table(entity)[ladder_clone_index_column(entity)].to_numpy() - # Each copy keeps its first clone, which carries both clones' weight. + # Each copy keeps its lowest-id clone, clone 0, which carries both + # clones' weight. assert kept.tolist() == (clone == 0).tolist() assert summed[kept].tolist() == [1.0] * 4 kept, summed = collapse_geographic_clones( @@ -125,8 +130,78 @@ def test_clones_that_differ_on_what_the_fit_reads_stay_rows_of_their_own() -> No assert np.isclose(summed[kept].sum(), 4.0) -#: The stages whose fits collapse clones, and the module each one's call is -#: in: the SPI income stage runs its forests from spi_income. +def _reversed_tables(frame) -> dict[str, pd.DataFrame]: + """Every table's rows in reverse order: each clone 1 before its clone 0.""" + + return { + name: table.iloc[::-1].reset_index(drop=True) + for name, table in _tables(frame).items() + } + + +def test_clones_in_reverse_row_order_keep_clone_zero() -> None: + """A record trains on its lowest-id clone, clone 0, wherever its row + sits: the UK entry orders the fold by the entity's ids, so a frame whose + clone 1 rows come first folds as the clone-major one does + (microcosm#931).""" + + tables = _reversed_tables(with_geographic_clones(_copies_frame())) + for entity in ("household", "person", "benunit"): + table = tables[entity] + n = len(table) + clone = table[ladder_clone_index_column(entity)].to_numpy() + assert clone[0] == 1 + rows, weights = np.ones(n, dtype=bool), np.full(n, 0.5) + kept, summed = collapse_geographic_clones( + tables, entity, rows, weights, inputs=_agreeing(n) + ) + assert kept.tolist() == (clone == 0).tolist(), entity + assert summed[kept].tolist() == [1.0] * 4 + fold = fold_geographic_clones( + tables, entity, rows, weights, inputs=_agreeing(n) + ) + assert fold.kept.tobytes() == kept.tobytes() + assert fold.weights.tobytes() == summed.tobytes() + # Every clone points at its own record's clone 0, the lowest id the + # record's clones carry. + ids = table[UK_NATIONAL_SCHEMA.entity_id_column(entity)].to_numpy() + assert (clone[fold.representative] == 0).all() + assert (fold.key_group[fold.representative] == fold.key_group).all() + for group in np.unique(fold.key_group): + members = fold.key_group == group + assert set(ids[fold.representative[members]]) == {ids[members].min()} + assert not fold.varied.any() + + +def test_a_variant_clone_trains_on_its_own_row_in_reverse_row_order() -> None: + """The record whose clone 1 differs keeps both rows, each its own + variant's representative, and only its rows vary; the other records + fold onto clone 0.""" + + tables = _reversed_tables(with_geographic_clones(_copies_frame())) + clone = tables["household"][ladder_clone_index_column("household")].to_numpy() + values = np.zeros(8) + differs = int(np.flatnonzero(clone == 1)[0]) + values[differs] = 1.0 + fold = fold_geographic_clones( + tables, + "household", + np.ones(8, dtype=bool), + np.full(8, 0.5), + inputs=pd.DataFrame({"value": values}), + ) + record = fold.key_group == fold.key_group[differs] + assert int(record.sum()) == 2 + assert fold.kept[record].all() + assert fold.representative[record].tolist() == np.flatnonzero(record).tolist() + assert fold.varied.tolist() == record.tolist() + assert fold.kept[~record].tolist() == (clone[~record] == 0).tolist() + assert fold.weights[fold.kept & ~record].tolist() == [1.0] * 3 + + +#: The stages whose fits fold clones, through either UK entry, and the module +#: each one's call is in: the SPI income stage runs its forests from +#: spi_income. _COLLAPSING_STAGES = { "hmrc_spi_income_spine": "spi_income", "spi_housing_shell": "spi_housing_shell", @@ -135,14 +210,16 @@ def test_clones_that_differ_on_what_the_fit_reads_stay_rows_of_their_own() -> No "salary_sacrifice": "salary_sacrifice", "spi_employment_shell": "spi_employment_shell", } +#: The UK entries a fit folds its clones through. +_FOLD_ENTRIES = frozenset({"collapse_geographic_clones", "fold_geographic_clones"}) -def _calls_collapse(path: Path) -> bool: +def _calls_fold(path: Path) -> bool: tree = ast.parse(path.read_text(encoding="utf-8")) return any( isinstance(node, ast.Call) and isinstance(node.func, ast.Name) - and node.func.id == "collapse_geographic_clones" + and node.func.id in _FOLD_ENTRIES for node in ast.walk(tree) ) @@ -155,7 +232,13 @@ def test_every_collapsing_stage_sees_the_lineage_live_at_it() -> None: includes the geographic clone index (microcosm#931).""" runtime = Path(inspect.getfile(graph_kernels)).parent - callers = {path.stem for path in runtime.glob("*.py") if _calls_collapse(path)} + # The module that defines the two entries calls one from the other. + entries = Path(inspect.getfile(fold_geographic_clones)).resolve() + callers = { + path.stem + for path in runtime.glob("*.py") + if path.resolve() != entries and _calls_fold(path) + } assert callers == set(_COLLAPSING_STAGES.values()) for stage, module in _COLLAPSING_STAGES.items(): names = { From 49f87c574ed116a8546a0c9ad52ce4d841625856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:28:13 +0000 Subject: [PATCH 058/108] Add the geographic expand and identity transforms, and make three UK stages ready for clones (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The geographic clone index columns' names, ladder_clone_index_column and ARTIFACT_CLONE_INDEX_COLUMN, and the clone count, geographic_clone_count, move unchanged to a new leaf module, uk_runtime/geographic_clones.py, which imports nothing first-party. rowwise_dataset and cgt_support re-export them, so every importer keeps working; household_lineage, salary_sacrifice, the new modules and the full build's population and export modules import them from the leaf. The leaf joins graph_kernels' counting modules, so every stage that counts by lineage hashes it. The CGT support split, which reads K through the count, binds it as a helper. The full build's kernels that read the names hash it beside rowwise_dataset: uk.full.expand and the other population kernels, uk.full.identity, uk.full.property.value, uk.full.private_rent and both export descriptions. The derived binding tables follow the move. The stage table's entry for the clone-index name retires, since every stage now reaches it through a hashed module. The kernel table keeps it at the leaf only for the readers that do not hash it, and the check of that entry's premise, that no spine node writes the clone index columns, moves beside it. geographic_expand.py adds G1, the step that gives every FRS household K geographic clones: expand_uk_geographic_clones(frame, clones=K) and its stage transform, UKGeographicExpandStageTransform(stage, clones). As María ruled on 10 Nov (decision 10), it calls the shared core, clone_expansion.expand_clones, with the UK's declaration, UK_GEOGRAPHIC_CLONES: - a CloneIdRule bounded by GEOGRAPHIC_CLONE_LIMIT, the geographic_support field's bound of 100, whose mint and require_below are named functions of the module that call copy_id_table on that field (mint_geographic_clone_ids, require_room_for_geographic_clones), so the derived binding rule binds the table to the key of the stage that runs G1; - a clone index on the person, benefit unit and household tables; - design weights only, and a mass record reason that names no K; - source_household_id, each household's clone-0 id. Clone k takes id + k x 10^9 on the five id columns and w / K of its household's design weight. The UK wrapper makes the refusals the core does not: - weights on any entity but the household, since the core splits every weighted entity and the UK weights households only; - a frame already carrying a clone index, a source column or a UK_COPY_LINEAGE copy column; - id or membership columns that are not int64, the dtype the copy-id table mints, at every K, so one clone takes the same frames as more; - a clone count outside 1 to 100; - a time period that is not a year. Before calling the core it writes source_year (the frame's year, int64) and source_household_key ("{source_year}:{source_household_id}", the values stage 16 writes, as Python strings in an object column, the dtype the legacy plan gives them) on the base household table, and it validates the frame as a UK national frame before and after. Its evidence gives K, the copy-id field and radix, the distinct source households rounded through receipt_counts, and the weight split and kind, with no row count, id or weight. Every refusal gives counts, names and declared constants only. geographic_identity.py adds G1b: key_uk_households_by_clone_path(frame) and UKGeographicIdentityStageTransform(stage). It writes geography_household_key as the dense line's uk.full.identity does: household_draw_key over the source frs, the FRS vintage frs_release.json pins for the household's source_year, its source_household_id and its copy path, read through source_lineage.copy_paths on UK_COPY_LINEAGE: () for clone 0 and (("geographic_support", k),) for clone k. The release maps its survey year and its base year, the frame's time period, to its vintage. The key is in the graph's string dtype, StringDtype("python"). An unmapped source year and repeated keys are refused by count, and no message prints a key. Its evidence gives the distinct source households rounded through receipt_counts, that the keys are unique, the source, the vintages and the columns it reads. Neither step is bound to a stage yet: no declaration, op kind, graph node or kernel names them, and no build module imports them. The next commit declares both and wires them at K = 1. Salary sacrifice draws each SPI-channel adult's bonus within its own geographic clone (O8a). draw_spi_bonuses takes clone_indices, which the stage reads as each person's household_clone_index; a frame without the column is one clone. The pay cells and the donor-source floor still pool every clone's donors, the floor counted in distinct source households. Each adult draws its bonus share from its own clone's paid donors in its cell, in id order, on the same person-keyed uniform, and a cell with no donor in the adult's clone is refused without a count. Where the clones agree on the donors' pay, clone 0 draws on the one-clone pool at w / K, up to the 1/K split's rounding, which can move a pooled cell edge where a quantile falls exactly on a cumulative weight share. UC capital counts a cell's donors in survey records (decision 6, María's ruling of 10 Nov). The stage folds its donors with fold_geographic_clones, and its widening test counts the distinct ReplicaFold.key_group values among a cell's donor rows instead of the rows. A unit's geographic clones count once, whether they fold into one row or donate apart as variants; any other copy, such as a high-value support copy, counts as a record of its own. Its comments and docstring now say a variant donates from its lowest-id clone. The fits' changelog fragment and docs/uk-atomic-household-lineage.md say so too. test_uk_clone_expansion_core.py now takes G1's declaration instead of a copy of its own and keeps clone_first_frame as its independent oracle, and the core's changelog fragment names G1 as its first caller. Why: the next commit should only declare and wire G1 and G1b, so both land here, held to the clone-first stand-in and to the dense identity kernel before any build runs them. The expansion core is shared (decision 10), so G1 is the UK's declaration and the refusals around it, not a UK copy of the core. The leaf lets the stages, G1 and G1b read the clone-index names, and stage 42 the count, without binding the dense line's rowwise_dataset into the spine, and an edit to it re-keys every stage and dense kernel that reads them. With clones ahead of stage 48, a pooled draw would hand an adult another clone's donor, so clone 0 would not draw as a one-clone spine does. Counted in rows, clones that differ would grow a UC capital cell's donor count with K and stop its widening sooner. The nesting test (O10) checks, stage by stage and before G1 is wired, what of a clone-first spine nests on the one-clone spine exactly and what only to rounding. What moves: node keys only. No value moves at one clone or on any spine: every spine is one clone today, and both stages give the parent commit's output bit for bit on invented frames, with or without a one-clone index. The bonus draw's declared text and UC capital's notes change with the declarations. These re-key: - every UK spine node. Every spine kernel (the root's CREATE, each stage, the identity and claim kernels and the spine gates) hashes graph_kernels, which this commit edits. The root and every other stage that counts by lineage also hash household_lineage, whose import moved, and now geographic_clones, which joins the counting modules. The CGT support split (stage 42) binds geographic_clones as a helper. Stages 26 and 42 hash cgt_support. Stage 36, and the six stages that bind it as a helper (34, 35 and 37 to 40), hash uc_capital_coherence. Stage 48 hashes salary_sacrifice. Keys chain, so in the in-graph build every node after the spine re-keys too; - on a bound spine, whose checkpoint kernel hashes none of these modules, the whole dense line after the checkpoint. Its population kernels (sample, normalize, expand, locations, geography mapping, pool and geography gate), uk.full.identity, uk.full.property.value, uk.full.private_rent and both export descriptions hash rowwise_dataset and graph_population, whose bytes change, and now geographic_clones; the identity, property value and private rent kernels hash household_lineage too; - on a bound spine, the national line from its problem on. The national problem and the calibration nodes hash graph_population, which this commit edits to bind geographic_clones into the population kernels' keys, and the national gates' registry modules include uc_capital_coherence. The national targets keep their key; - the full gates' terminal phase, whose own code hashes uc_capital_coherence and graph_population (gate_phase_codes.uk_terminal_only_code_hash). The spine gates' phase codes do not move. No node hashes geographic_expand, geographic_identity or the shared clone_expansion core until the next commit binds the two stages. From G1's transform, the derived rule already reaches the core and the copy-id table's mint, require_below, radix and ordinal_bound, through G1's named mint and require_below. In the binding tables: - the stage table drops rowwise_dataset's ladder_clone_index_column, which every spine node now binds through the root; - the kernel table lists the name at geographic_clones only for the readers that do not hash it: the calibrated gates' terminal phase, the dense measures, the national problem and the national gates. uk.full.expand@1 now hashes it; - stage 42's own hash gains geographic_clones; - the copy-id table's readers gain geographic_expand. No generated file, declaration or gates.json changes, since uk_spine.json and the H2 fixture hold stage contract hashes, not code hashes. No transport kernel and no US, NZ or BE node hashes an edited module, so their node keys stay. Only the US PUF-support builder's whole-tree resume fingerprint moves, as it does on any edit to packaged source. Tests (engine-free; invented values only): - test_uk_geographic_expand.py: G1 equals clone_first_frame bit for bit at K = 1, 2, 3, 7 and 15; one clone adds no row and only the declared columns, and keeps the mass log it is handed with one record more; the clone-major ids, indices, sources and weights at K = 3; the source key's dtype under either string inference; the K-free mass record; the evidence; the declaration's named functions; and each refusal, the int64 one at one clone and at two, with no id or weight in its message. - test_uk_geographic_identity.py: every clone's key equals the dense identity kernel's at K = 1 and 3; the clone path; unique keys that nest across K; the dtype; the vintage for the release's survey and base years; the evidence, in distinct source households at every K; and each refusal, with no key or id in its message. - test_uk_clone_first_nesting.py (O10): an invented frame through G1 and stages 16, 17 and 42 at K = 1, 2, 3, 7 and 15. Stage 17's mass scale is reported first. Clone 0 equals the one-clone frame on its ids, G1's columns, stage 16's copies and stage 17's seats and carriers, and every frame and copy keeps the clone-index invariant. K x each clone-0 weight is held to one clone's: rows an exact-total correction moved within rows x K x 2.2e-16 relative; at stage 17, the other incumbents of a funding stratum a stage-16 correction reached within the stage's step budget at every K; other rows bitwise at K = 2, and at K = 3, 7 and 15 within one ulp where the 1/K split is their only rounding, else one ulp per weight-setting step. Stage 42 nests approximately (family masses, the divided band, the crossed band's mass, rule B), and through G1 rule B's correction lands on a copy at 7 and 13 clones. - test_uk_salary_sacrifice.py: a one-clone index draws as none; three clones keep the one-clone cells, clone 0 draws as one clone does and each adult takes a donor of its own clone; the empty cell-clone refusal holds no digit; and the stage at one and three clones. - test_uk_uc_capital_coherence.py: a one-clone index redraws as none; at three clones that differ, the coarsening levels are three times one clone's; and a high-value copy counts as a donor of its own. - test_uk_cgt_support.py: every spelling of the household clone index equals the leaf's, and the re-exports are the leaf's own objects. Its nesting scenario's data move to test_support/microcosm_build/ uk_cgt_support.py, which the nesting test shares. - test_uk_copy_id_table.py lists geographic_expand among the copy-id readers, and test_uk_graph.py re-derives the binding tables and checks that an edit to the leaf re-keys every counting stage. These files, the whole of test_uk_graph.py, test_uk_graph_terminal.py, test_uk_cgt_structure.py, test_uk_receipt_counts.py, test_uk_geographic_clone_collapse.py, test_uk_atomic_identity_graph.py, test_first_party_imports.py and test_clone_expansion.py pass. tools/ci_test_plan.py verify is ok, and ruff check and ruff format --check are clean. Co-Authored-By: Claude Opus 5.5 --- .../931-shared-clone-expansion-core.added.md | 2 +- ...31-uk-fits-fold-geographic-clones.added.md | 2 +- .../931-uk-stages-ready-for-clones.changed.md | 51 + docs/uk-atomic-household-lineage.md | 2 +- .../microcosm/build/uk_runtime/cgt_support.py | 60 +- .../build/uk_runtime/geographic_clones.py | 93 ++ .../build/uk_runtime/geographic_expand.py | 400 ++++++ .../build/uk_runtime/geographic_identity.py | 285 +++++ .../build/uk_runtime/graph_kernels.py | 22 +- .../build/uk_runtime/graph_population.py | 32 +- .../build/uk_runtime/graph_terminal.py | 11 +- .../build/uk_runtime/household_lineage.py | 2 +- .../build/uk_runtime/inspection_export.py | 9 +- .../build/uk_runtime/rowwise_dataset.py | 25 +- .../build/uk_runtime/salary_sacrifice.py | 80 +- .../microcosm/build/uk_runtime/spi_support.py | 2 +- .../build/uk_runtime/uc_capital_coherence.py | 49 +- .../engine_free/uk/test_uk_cgt_support.py | 60 +- .../uk/test_uk_clone_expansion_core.py | 50 +- .../uk/test_uk_clone_first_nesting.py | 1072 +++++++++++++++++ .../engine_free/uk/test_uk_copy_id_table.py | 6 +- .../uk/test_uk_geographic_expand.py | 552 +++++++++ .../uk/test_uk_geographic_identity.py | 409 +++++++ .../tests/engine_free/uk/test_uk_graph.py | 41 +- .../uk/test_uk_salary_sacrifice.py | 196 ++- .../uk/test_uk_uc_capital_coherence.py | 126 +- .../microcosm_build/uk_cgt_support.py | 26 + 27 files changed, 3458 insertions(+), 207 deletions(-) create mode 100644 changelog.d/931-uk-stages-ready-for-clones.changed.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_expand.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_identity.py diff --git a/changelog.d/931-shared-clone-expansion-core.added.md b/changelog.d/931-shared-clone-expansion-core.added.md index 410fc8a75c..efc7a11e99 100644 --- a/changelog.d/931-shared-clone-expansion-core.added.md +++ b/changelog.d/931-shared-clone-expansion-core.added.md @@ -25,4 +25,4 @@ A country-neutral clone expansion core, `microcosm.build.clone_expansion` (#931; - **Tests.** - `tests/engine_free/shared/test_clone_expansion.py` runs the core under neutral and US-shaped declarations on invented frames. The US-shaped frame weights households and people, each split with a record of its own, and its clones, labelled from their clone index, pass the shared two-channel invariant `outer_stage_runtime.assert_clone_expansion`. Narrow ids keep their dtype at every K, and each refusal, the dtype ones included, is checked to print no id or weight. - `tests/engine_free/uk/test_uk_clone_expansion_core.py` runs it under the UK's declarations: the copy-id table's `geographic_support` mint and `require_below`, the ladder's clone-index columns, `source_household_id`, and design weights on households. At K = 1, 2, 3, 7 and 15 it reproduces the clone-first stand-in `clone_first_frame`, which is written without the core, bit for bit on the ids, the household clone index, the source household id and the weights. -- **No caller yet.** No build module imports it yet, so no output and no node key moves. +- **First caller.** The UK's geographic expansion step (`microcosm.build.uk_runtime.geographic_expand`, G1 of the clone-first spine) calls the core with the UK's declaration, `UK_GEOGRAPHIC_CLONES`. Its mint and its `require_below` are named functions in that module that call the copy-id table, so the derived binding rule binds the table to the key of the stage that runs the step. The UK test above now takes that declaration from the step instead of keeping a copy of its own. No stage runs the step yet, so the core moves no output and no node key. diff --git a/changelog.d/931-uk-fits-fold-geographic-clones.added.md b/changelog.d/931-uk-fits-fold-geographic-clones.added.md index c83f671259..88db8e49e7 100644 --- a/changelog.d/931-uk-fits-fold-geographic-clones.added.md +++ b/changelog.d/931-uk-fits-fold-geographic-clones.added.md @@ -5,7 +5,7 @@ The UK fits trained on the frame's FRS rows now fold geographic clones into each - **What changes once the spine clones before enrichment:** - the fits see each record once wherever its clones agree, and every clone's value at its share wherever they differ; - the reference transport meets no duplicate twins from agreeing clones; - - the UC capital cells' donor minimum counts records, not identical clones. + - the UC capital cells' donor minimum counts survey records, so neither identical nor differing clones grow it. - **The country-neutral pieces.** `microcosm.build.source_lineage` gains three functions: - `copy_paths`: every household's ordered copy path from a declared lineage, never inferred from ids, weights or values; - `replica_keys`: a key per record of any entity, shared only by its replicas in the named branches, with a member's rank in its household telling members apart; diff --git a/changelog.d/931-uk-stages-ready-for-clones.changed.md b/changelog.d/931-uk-stages-ready-for-clones.changed.md new file mode 100644 index 0000000000..11c6b02338 --- /dev/null +++ b/changelog.d/931-uk-stages-ready-for-clones.changed.md @@ -0,0 +1,51 @@ +The UK spine gains G1 and G1b, the steps that will give it geographic clones, without wiring them yet, and three of its stages are made ready for clones: salary sacrifice draws SPI bonuses within each clone, UC capital counts its donors in survey records, and the CGT support split reads the clone count from a module of its own (#931). No output moves: every spine is one clone today. + +- **Clone-index names.** `ladder_clone_index_column`, `ARTIFACT_CLONE_INDEX_COLUMN` and `geographic_clone_count` move unchanged to `microcosm.build.uk_runtime.geographic_clones`, which imports nothing first-party. `rowwise_dataset` and `cgt_support` re-export them, so every importer keeps working. The module joins the spine's counting modules, so every stage that counts by lineage hashes it. The CGT support split, which reads the clone count, binds it as a helper. The full build's population and export kernels, which read the names, hash it too: `uk.full.expand` and the other population kernels, `uk.full.identity`, `uk.full.property.value`, `uk.full.private_rent` and both export descriptions. +- **Geographic expansion (G1).** `geographic_expand.expand_uk_geographic_clones(frame, clones=K)` and its stage transform, `UKGeographicExpandStageTransform(stage, clones)`, turn a spine frame into K clones of every FRS household, clone-major. They call the shared clone expansion core, as María ruled on 10 Nov, with the UK's declaration, `UK_GEOGRAPHIC_CLONES`: + - clone `k` takes `id + k × 10^9` on all five id columns, in the copy-id table's `geographic_support` field, through two named functions that call the table (`mint_geographic_clone_ids`, `require_room_for_geographic_clones`); clone 0 keeps the frame's ids; + - each table carries its clone index (`person_clone_index`, `benunit_clone_index`, `household_clone_index`), 0 to K - 1, with exactly K values; + - each clone takes `w / K` of its household's design weight, the same float in every clone, and the mass log gains one record whose reason names no clone count; + - each household carries the SPI support channel's source columns: `source_year` (the frame's time period) and `source_household_key` (`"{source_year}:{source_household_id}"`, the values that channel writes, as Python strings in an object column, the dtype the legacy plan gives it; the graph casts it to its string dtype), written before the clones are made, and `source_household_id`, its clone-0 id, which the core writes. + + It refuses, by name or by count only: + - a clone count outside 1 to 100 (`GEOGRAPHIC_CLONE_LIMIT`, the field's bound); + - weights on any entity but the household, or of any kind but design; + - a frame already carrying a clone index, a source column or a column of the UK copy lineage; + - id or membership columns that are not int64, the dtype the table mints, at every clone count, so one clone takes the same frames as more; + - a time period that is not a year; + - incoming ids at or above the field's radix; + - whatever the core refuses. + + Its evidence gives K, the copy-id field and its radix, the distinct source households rounded through `receipt_counts` at the floor of three, and the weight split and kind, with no row count, id or weight. +- **Geography identity (G1b).** `geographic_identity.key_uk_households_by_clone_path(frame)` and `UKGeographicIdentityStageTransform(stage)` give every household of G1's frame the key the dense line's `uk.full.identity` writes, `geography_household_key`. The key is `household_draw_key` over: + - the source `frs`; + - the FRS vintage frs_release.json pins for the household's `source_year`, which maps the release's survey year and its base year, the frame's time period, to its vintage; + - its `source_household_id`; + - its copy path, read from the lineage columns through `source_lineage.copy_paths`: empty for clone 0, and the `geographic_support` branch at `k` for clone `k`. + + So each clone keys as the dense identity keys that clone, and a larger K keeps every key. The key is in the graph's `string` dtype (`StringDtype("python")`). It refuses, by count or by name and never printing a key, a frame that already carries the key or lacks G1's int64 columns, a source year frs_release.json pins no vintage for, and repeated keys. Its evidence gives the distinct source households rounded through `receipt_counts`, that the keys are unique, the source, the vintages and the columns it reads. +- **Not wired yet (G1, G1b).** No declaration, op kind, graph node or kernel names G1 or G1b, and no build module imports them, so neither runs in a build. They move no output and no node key until they are declared. +- **SPI bonuses per clone (salary sacrifice).** `draw_spi_bonuses` takes `clone_indices`, each row's geographic clone. The stage passes each person's household `household_clone_index`, and a frame without the column is one clone. + - The pay cells and the donor-source floor still pool every clone's donors, the floor counted in distinct source households. + - Each SPI-channel adult now draws its bonus share from the donors of its own clone in its cell, in id order, on the same person-keyed uniform. + - At one clone the draw is unchanged, value for value. Where the clones agree on the donors' pay, clone 0 of K clones draws on the one-clone pool at `w / K`, up to the 1/K split's rounding, which can move a pooled cell edge where a quantile falls exactly on a cumulative weight share. + - A cell with no donor in an SPI adult's clone is refused, with no count, id or value in the message. + - The declared draw text (`spi_bonus_draw`) changes with the declarations. +- **UC capital donors counted in survey records** (María's ruling of 10 Nov). The stage folds its donors through `fold_geographic_clones`, and a cell stops widening once it holds `minimum_cell_donors` survey records: the distinct replica keys (`ReplicaFold.key_group`) among its donor rows, not the rows. + - A unit's geographic clones count once, whether they fold into one row or differ and donate apart. Any other copy, such as a high-value support copy, counts as a record of its own. + - A donor variant donates from its lowest-id clone. + - Without geographic clones every row is its own record, so the count and every draw are unchanged. + - The stage's notes change with the declarations. +- **What moves.** No output, at one clone or on any spine: both stages give the parent commit's output bit for bit on invented frames, with or without a one-clone index. Node keys move: + - every UK spine node. Every spine kernel hashes `graph_kernels`, which this commit edits, and the stages that count by lineage now also hash `geographic_clones`. Keys chain, so in the in-graph build every node after the spine moves too; + - on a bound spine, whose checkpoint kernel hashes no edited module, the whole dense line after the checkpoint: its population kernels hash `rowwise_dataset`, `graph_population` and now `geographic_clones`; + - on a bound spine, the national line from its problem on. The national problem and the calibration nodes hash `graph_population`, which this commit edits to bind `geographic_clones` into the population kernels' keys, and the national gates' registry modules include `uc_capital_coherence`. The national targets keep their key; + - the full gates' terminal phase code, which hashes `uc_capital_coherence` and `graph_population`. + + No generated file, declaration or `gates.json` changes. No transport kernel and no US, NZ or BE node hashes an edited module; only the US PUF-support builder's whole-tree resume fingerprint moves, as on any edit to packaged source. +- **Tests.** + - `test_uk_geographic_expand.py` holds G1 to the clone-first stand-in `clone_first_frame` bit for bit at K = 1, 2, 3, 7 and 15. It also checks the one-clone frame and the mass log it is handed, the layout at K = 3, the source key's dtype under either string inference, the mass record, the evidence, the declaration's named functions and each refusal, the int64 one at one clone and at two. + - `test_uk_geographic_identity.py` holds every clone's key to the dense identity kernel's at K = 1 and 3. It also checks the clone path, uniqueness and nesting across K, the dtype, the vintage for the release's survey and base years, the evidence and each refusal. + - `test_uk_clone_first_nesting.py` runs an invented frame through G1 and stages 16, 17 and 42 at K = 1, 2, 3, 7 and 15. Clone 0 equals the one-clone frame on its ids, G1's columns, stage 16's copies and stage 17's seats, and every frame and copy keeps the clone-index invariant. K × each clone-0 weight matches one clone's within stated rounding classes, stage 42 nests approximately, and through G1 rule B's correction lands on a copy at 7 and 13 clones. + - The salary-sacrifice and UC capital tests cover the per-clone draw and its refusal, the stage at one and three clones, a one-clone index, clones that differ, and a high-value copy. + - `test_uk_cgt_support.py` holds every spelling of the household clone index to the module's name, and its nesting scenario's data move to `test_support` for the nesting test to share. `test_uk_clone_expansion_core.py` takes G1's declaration, and `test_uk_copy_id_table.py` lists `geographic_expand` among the copy-id readers. `test_uk_graph.py` re-derives the binding tables, checks that no spine node writes the clone index columns, and checks that an edit to `geographic_clones` re-keys every counting stage. diff --git a/docs/uk-atomic-household-lineage.md b/docs/uk-atomic-household-lineage.md index 1953e1a6d7..101ff7e544 100644 --- a/docs/uk-atomic-household-lineage.md +++ b/docs/uk-atomic-household-lineage.md @@ -29,7 +29,7 @@ The UK's copy lineage is declared once, as data, in `uk_runtime.household_lineag - `record_sources` gives every record of any entity its true source through household membership. Every UK floor counts distinct sources through it (`receipt_counts.record_source_households`), so a household's copies and clones count once. - `copy_paths` reads every household's ordered clone path; `uk_household_clone_paths` is the UK's strict entry for the identity kernel. -- `replica_keys` and `fold_replicas` keep one training row per true source and path without the geographic branch, with the clones' weights summed; through the UK entries each variant trains on its lowest-id clone, whatever the row order. `collapse_replicas` returns the fold's rows and weights alone. The FRS-trained fits fold their training rows through `household_lineage.collapse_geographic_clones`: the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. Once geographic clones precede enrichment, each fit still sees each survey record once. A frame without geographic clones keeps its rows and weights value for value. +- `replica_keys` and `fold_replicas` keep one training row per true source and path without the geographic branch, with the clones' weights summed; through the UK entries each variant trains on its lowest-id clone, whatever the row order. `collapse_replicas` returns the fold's rows and weights alone. The FRS-trained fits fold their training rows through the UK entries (`household_lineage.collapse_geographic_clones`, or `fold_geographic_clones` where a stage reads the whole fold, as UC capital does to count its cells' donors in survey records): the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. Once geographic clones precede enrichment, each fit still sees each survey record once. A frame without geographic clones keeps its rows and weights value for value. - Each fit passes what it reads (its predictors and targets, or the columns they derive from), and clones fold only where they agree on it. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own. Each variant carries the summed weight of the clones that share it, so the fit sees every clone's value at its share and nothing is dropped. ## Verification scope diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index 541cee915e..ae5ca0d14e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -52,6 +52,11 @@ _assert_closed_world_operations, _oldest_adult_indices, ) + +# Moved to geographic_clones, beside the clone index columns' names; this +# stage's node hashes that module (microcosm#931). Kept importable from here +# for this module's other callers. +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_CONDITIONING_RESOURCE, HMRC_CGT_INCOME_BAND_LOWER_BOUNDS, @@ -173,10 +178,11 @@ ) _HOUSEHOLD_SUPPORT_CHANNEL_COLUMN = "household_support_channel" #: The geographic clone index the clone-first spine gives every household -#: (``rowwise_dataset.ladder_clone_index_column("household")``, the -#: ``geographic_support`` branch of the UK copy lineage; microcosm#931). It is -#: spelled here, not imported, so a change to it re-keys this stage; a test -#: holds the two spellings equal. +#: (``geographic_clones.ladder_clone_index_column("household")``, the +#: ``geographic_support`` branch of the UK copy lineage; microcosm#931). This +#: stage reads ``K`` through ``geographic_clones.geographic_clone_count``, +#: which reads that module's name, and its node hashes that module, so no code +#: here reads this spelling any more; a test holds it equal to the others. _GEOGRAPHIC_CLONE_INDEX_COLUMN = "household_clone_index" _PERSON_ID_COLUMNS = ("person_id", "person_household_id", "person_benunit_id") _POLICY_PARAMETER_NAMES = ( @@ -508,52 +514,6 @@ def _require_clone_count(clone_count: object) -> int: return int(clone_count) -def geographic_clone_count(household: pd.DataFrame) -> int: - """The frame's geographic clone count ``K``: one more than its largest index. - - The clone-first spine (microcosm#931) gives every household the index of - its geographic clone in ``household_clone_index``, 0 to ``K - 1``; a - household table without that column, as on every national spine today, - is one clone. Refuses a non-integer index column, a household without an - index, a negative index and indices that are not exactly 0 to ``K - 1`` - (an index below the largest that no household carries). Each refusal - gives a count or the column's dtype, never an id or a value. - """ - - column = _GEOGRAPHIC_CLONE_INDEX_COLUMN - if column not in household.columns: - return 1 - values = household[column] - if pd.api.types.is_bool_dtype(values.dtype) or not ( - pd.api.types.is_integer_dtype(values.dtype) - ): - raise ValueError( - "The CGT support split counts geographic clones on an integer " - f"{column!r} column; it has dtype {values.dtype}." - ) - rule = ( - f"the CGT support split counts K geographic clones as the {column!r} " - "values 0 to K - 1 (microcosm#931)." - ) - unindexed = int(values.isna().sum()) - if unindexed: - raise ValueError(f"{unindexed} household(s) carry no {column!r}; {rule}") - indices = values.to_numpy(dtype=np.int64) - if not indices.size: - return 1 - negative = int((indices < 0).sum()) - if negative: - raise ValueError(f"{negative} household(s) carry a negative {column!r}; {rule}") - clone_count = int(indices.max()) + 1 - absent = clone_count - int(np.unique(indices).size) - if absent: - raise ValueError( - f"{absent} of the {clone_count} indices from 0 to the largest are " - f"carried by no household; {rule}" - ) - return clone_count - - def cgt_support_copy_counts( weights: Sequence[float] | np.ndarray, maximum_copy_weight: float = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py new file mode 100644 index 0000000000..0cac7cace6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py @@ -0,0 +1,93 @@ +"""The UK geographic clone index columns' names, and a frame's clone count. + +Each geographic clone of a household carries its index, 0 to ``K - 1``, on +the household, its benefit units and its people, under a per-entity name +(:func:`ladder_clone_index_column`); the written artifact keeps the legacy +schema's one shared name (:data:`ARTIFACT_CLONE_INDEX_COLUMN`). The UK copy +lineage reads the household's index as its ``geographic_support`` branch +(``household_lineage.UK_COPY_LINEAGE``), and :func:`geographic_clone_count` +reads ``K`` from it. + +The two names lived in ``rowwise_dataset`` and the count in ``cgt_support``, +which re-export them. This module holds the three alone and imports nothing +first-party, as ``annuity_trust_column`` holds its column's name. The stages +that count by lineage read the household's name through ``household_lineage``, +and their nodes hash this module with the counting modules +(``graph_kernels._COUNTING_MODULES``); the CGT support split, which reads +``K`` through the count, hashes it too. The full build's population and export +kernels, which read the names, hash it as well. So an edit here re-keys all +of them, without binding the dense line's ``rowwise_dataset`` into the spine +(microcosm#931). +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +__all__ = [ + "ARTIFACT_CLONE_INDEX_COLUMN", + "geographic_clone_count", + "ladder_clone_index_column", +] + +#: Clone-index column name in the written artifact (one name per table — +#: the legacy single-year schema, unchanged by the Frame carrier). +ARTIFACT_CLONE_INDEX_COLUMN = "clone_index" + + +def ladder_clone_index_column(entity: str) -> str: + """The in-memory, per-entity clone-index column on the ladder frame. + + Frame enforces globally unique column names across entity tables, so the + artifact's shared ``clone_index`` name cannot appear on all three tables + in memory; the writer renames these back at the export boundary. + """ + + return f"{entity}_{ARTIFACT_CLONE_INDEX_COLUMN}" + + +def geographic_clone_count(household: pd.DataFrame) -> int: + """The frame's geographic clone count ``K``: one more than its largest index. + + The clone-first spine (microcosm#931) gives every household the index of + its geographic clone in ``household_clone_index``, 0 to ``K - 1``; a + household table without that column, as on every national spine today, + is one clone. Refuses a non-integer index column, a household without an + index, a negative index and indices that are not exactly 0 to ``K - 1`` + (an index below the largest that no household carries). Each refusal + gives a count or the column's dtype, never an id or a value. + """ + + column = ladder_clone_index_column("household") + if column not in household.columns: + return 1 + values = household[column] + if pd.api.types.is_bool_dtype(values.dtype) or not ( + pd.api.types.is_integer_dtype(values.dtype) + ): + raise ValueError( + "The CGT support split counts geographic clones on an integer " + f"{column!r} column; it has dtype {values.dtype}." + ) + rule = ( + f"the CGT support split counts K geographic clones as the {column!r} " + "values 0 to K - 1 (microcosm#931)." + ) + unindexed = int(values.isna().sum()) + if unindexed: + raise ValueError(f"{unindexed} household(s) carry no {column!r}; {rule}") + indices = values.to_numpy(dtype=np.int64) + if not indices.size: + return 1 + negative = int((indices < 0).sum()) + if negative: + raise ValueError(f"{negative} household(s) carry a negative {column!r}; {rule}") + clone_count = int(indices.max()) + 1 + absent = clone_count - int(np.unique(indices).size) + if absent: + raise ValueError( + f"{absent} of the {clone_count} indices from 0 to the largest are " + f"carried by no household; {rule}" + ) + return clone_count diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py new file mode 100644 index 0000000000..55c3d38414 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py @@ -0,0 +1,400 @@ +"""The UK spine's geographic clones: K copies of every FRS household (microcosm#931). + +The clone-first spine gives every FRS household ``K`` geographic clones before +any copy step runs. Each clone carries an equal share of its household's +design weight, so a later stage can give each clone an area of its own while +together they stand for the survey household once. This module is that step, +G1 of the clone-first plan: :class:`UKGeographicExpandStageTransform` and the +function it runs, :func:`expand_uk_geographic_clones`. + +The clones come from the shared, country-neutral core +(:func:`microcosm.build.clone_expansion.expand_clones`; shared rather than +UK-local, María's ruling of 2026-11-10) under the UK's declaration, +:data:`UK_GEOGRAPHIC_CLONES`: + +- **Ids.** Clone ``k`` of a row takes ``id + k x radix`` on all five id + columns, in the copy-id table's ``geographic_support`` field + (:func:`mint_geographic_clone_ids`). So a clone's id is a function of its + source id and its clone alone, and clone 0 keeps the frame's own ids. + Incoming ids the field has no room for are refused first + (:func:`require_room_for_geographic_clones`). +- **Clone indices.** Each table's ``{entity}_clone_index``, 0 to ``K - 1``, + with exactly ``K`` values on every table. +- **Weights.** Design weights only. Each clone takes ``w / K`` of its + household's weight, the same float in every clone, and the mass log gains + one record whose reason names no clone count. +- **Source id.** ``source_household_id``, the clone-0 id. + +The UK wrapper adds what the core does not do: + +- It refuses a frame that already carries a clone index, a source column or + any of the UK copy lineage's columns, since the step runs once, before any + copy. It also refuses weights on any entity but the household: the core + splits every weighted entity, and the UK weights households only. And it + refuses id and membership columns that are not int64, the dtype the + copy-id table mints, at every clone count, so every clone count takes the + same frames. +- Before the core runs, it writes each household's ``source_year`` (the + frame's time period, as the SPI support channel reads it) and + ``source_household_key`` (``f"{source_year}:{source_household_id}"``, the + values that stage writes, as Python strings in an object column) on the + frame's own rows, so every clone carries its household's. +- It validates the clones as a UK national frame. + +Every refusal gives counts, entities, columns, dtypes and declared constants +only, never an id, a weight or another value of a row. The evidence gives +``K``, the copy-id field and its radix, the distinct source households +rounded through ``receipt_counts``, and the weight split and kind; no row +count, id or weight. + +No stage runs this step yet, so it moves no output and no node key. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd + +from microcosm.build.clone_expansion import ( + CloneDeclaration, + CloneIdRule, + expand_clones, +) +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE +from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.receipt_counts import ( + distinct_carriers, + record_source_households, + rounded_count, +) +from microcosm.frame import Frame, WeightKind +from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN + +__all__ = [ + "GEOGRAPHIC_CLONE_LIMIT", + "GEOGRAPHIC_COPY_ID_FIELD", + "GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON", + "GEOGRAPHIC_EXPAND_STAGE_NAME", + "GEOGRAPHIC_EXPAND_WEIGHT_SPLIT", + "SOURCE_HOUSEHOLD_KEY_COLUMN", + "SOURCE_YEAR_COLUMN", + "UK_GEOGRAPHIC_CLONES", + "UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS", + "UK_GEOGRAPHIC_SOURCE_COLUMNS", + "UKGeographicExpandResult", + "UKGeographicExpandStageTransform", + "expand_uk_geographic_clones", + "mint_geographic_clone_ids", + "require_room_for_geographic_clones", +] + +GEOGRAPHIC_EXPAND_STAGE_NAME = "geographic_expand" +#: The copy-id table's field the clones' ids are minted in. +GEOGRAPHIC_COPY_ID_FIELD = "geographic_support" +#: The most clones the field numbers: clone ordinals run below it. Exported +#: so the spine and its graph can read the limit from here. +GEOGRAPHIC_CLONE_LIMIT = copy_id_table.ordinal_bound(GEOGRAPHIC_COPY_ID_FIELD) +#: The household's survey year, which the step writes on every household. +SOURCE_YEAR_COLUMN = "source_year" +#: ``f"{source_year}:{source_household_id}"``, which the step writes on every +#: household. +SOURCE_HOUSEHOLD_KEY_COLUMN = "source_household_key" +#: The source columns the step writes, which a frame entering it must lack. +UK_GEOGRAPHIC_SOURCE_COLUMNS = ( + SOURCE_HOUSEHOLD_ID_COLUMN, + SOURCE_YEAR_COLUMN, + SOURCE_HOUSEHOLD_KEY_COLUMN, +) +#: Every column the step adds: each table's clone index, then the household's +#: source columns. +UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS = ( + *(ladder_clone_index_column(entity) for entity in UK_NATIONAL_SCHEMA.entities), + *UK_GEOGRAPHIC_SOURCE_COLUMNS, +) +#: The reason of the step's mass record. It names no clone count, so it is the +#: same at every K. +GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON = ( + "Geographic clones split each household's design weight equally among its " + "clones, so household mass is conserved." +) +#: How the clones share a household's weight: each takes an equal share. +GEOGRAPHIC_EXPAND_WEIGHT_SPLIT = "equal" + + +def mint_geographic_clone_ids(ids: np.ndarray, k: int) -> np.ndarray: + """Clone ``k``'s ids: each id plus ``k`` times the ``geographic_support`` + field's radix, through the copy-id table, which refuses an ordinal outside + the field, an id already in it and an int64 overflow, by count.""" + + return copy_id_table.mint(ids, GEOGRAPHIC_COPY_ID_FIELD, k) + + +def require_room_for_geographic_clones(ids: np.ndarray, what: str) -> None: + """Refuse incoming ids at or above the ``geographic_support`` field's + radix, printing only how many, through the copy-id table.""" + + copy_id_table.require_below(ids, GEOGRAPHIC_COPY_ID_FIELD, what=what) + + +#: The UK's declaration for the shared clone expansion core: ids minted in the +#: ``geographic_support`` field, a clone index on every table, design weights, +#: the step's mass record reason and the household's clone-0 id. +UK_GEOGRAPHIC_CLONES = CloneDeclaration( + ids=CloneIdRule( + bound=GEOGRAPHIC_CLONE_LIMIT, + mint=mint_geographic_clone_ids, + require_below=require_room_for_geographic_clones, + ), + clone_index_columns={ + entity: ladder_clone_index_column(entity) + for entity in UK_NATIONAL_SCHEMA.entities + }, + weight_kinds=frozenset({WeightKind.DESIGN}), + reason=GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON, + source_id_columns={"household": SOURCE_HOUSEHOLD_ID_COLUMN}, +) +#: The columns a frame entering the step must lack: what the step writes, and +#: every copy step's columns, since the step runs before any copy. +_RESERVED_COLUMNS = tuple( + dict.fromkeys((*UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, *UK_COPY_LINEAGE.copy_columns)) +) + + +@dataclass(frozen=True) +class UKGeographicExpandResult: + """The clones and what the step's evidence reads of them. + + ``source_households`` is the exact count of distinct source households; + the evidence gives it rounded, and neither field prints in the repr. + """ + + frame: Frame = field(repr=False) + clone_count: int + source_households: int = field(repr=False) + + def evidence(self) -> dict[str, object]: + return { + "stage": GEOGRAPHIC_EXPAND_STAGE_NAME, + "clone_count": int(self.clone_count), + "copy_id_field": GEOGRAPHIC_COPY_ID_FIELD, + "copy_id_radix": copy_id_table.radix(GEOGRAPHIC_COPY_ID_FIELD), + "source_households": rounded_count(self.source_households), + "weight_split": GEOGRAPHIC_EXPAND_WEIGHT_SPLIT, + "weight_kind": WeightKind.DESIGN.value, + } + + +def _clone_count(clones: object) -> int: + """``clones`` as an int, refusing anything outside [1, the limit].""" + + if ( + isinstance(clones, bool) + or not isinstance(clones, int | np.integer) + or not 1 <= clones <= GEOGRAPHIC_CLONE_LIMIT + ): + raise ValueError( + "The geographic expansion takes a clone count from 1 to " + f"{GEOGRAPHIC_CLONE_LIMIT}, the {GEOGRAPHIC_COPY_ID_FIELD!r} copy-id " + "field's declared bound (microcosm#931)." + ) + return int(clones) + + +def _assert_geographic_expand_stage(stage: SourceStageSpec) -> None: + """The stage a transform runs as: the step's name, household grain, and + exactly the step's columns as outputs, with no rewrites.""" + + if not isinstance(stage, SourceStageSpec): + raise TypeError("The geographic expansion runs as a SourceStageSpec stage.") + if stage.stage != GEOGRAPHIC_EXPAND_STAGE_NAME: + raise ValueError( + f"Expected stage {GEOGRAPHIC_EXPAND_STAGE_NAME!r}, got {stage.stage!r}." + ) + if stage.grain != "household": + raise ValueError( + "The geographic expansion clones households; grain must be " + f"'household', got {stage.grain!r}." + ) + if tuple(stage.outputs) != UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS or tuple( + stage.rewrites + ): + raise ValueError( + "The geographic expansion writes exactly the clone indices and the " + "source columns; the stage must declare outputs " + f"{UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS} and no rewrites, got " + f"{tuple(stage.outputs)} / {tuple(stage.rewrites)}." + ) + + +def _refuse_other_weighted_entities(frame: Frame) -> None: + weighted = tuple(frame.weighted_entities) + if weighted != ("household",): + raise ValueError( + "The geographic expansion splits household weights alone, as the UK " + f"weights households only; the frame weights {list(weighted)}." + ) + + +def _refuse_narrow_ids(frame: Frame) -> None: + """Refuse id and membership columns that are not int64. + + The copy-id table mints int64 ids, so the core refuses narrower ids from + two clones on; refusing them at one clone too keeps every clone count + taking the same frames. + """ + + schema = frame.schema + person = frame.table(schema.person_entity) + columns = [(frame.table(e), schema.entity_id_column(e)) for e in frame.entities] + columns += [(person, schema.membership_column(g)) for g in schema.group_entities] + narrow = [ + f"{column!r} holds {table[column].dtype}" + for table, column in columns + if table[column].dtype != np.dtype("int64") + ] + if narrow: + raise ValueError( + "The geographic expansion takes int64 id and membership columns, " + f"the dtype the copy-id table mints; {', '.join(narrow)}." + ) + + +def _refuse_carried_lineage(frame: Frame) -> None: + carried = set().union(*(frame.table(entity).columns for entity in frame.entities)) + found = [column for column in _RESERVED_COLUMNS if column in carried] + if found: + raise ValueError( + "The geographic expansion clones a frame once, before any copy " + f"step; the frame already carries {found}." + ) + + +def _source_year(frame: Frame) -> int: + """The frame's time period as a year, as the SPI support channel reads it.""" + + try: + return int(uk_time_period(frame)) + except ValueError as error: + raise ValueError( + "The geographic expansion writes each household's source_year from " + "the frame's time period, which must name a year." + ) from error + + +def _with_source_year_and_key(frame: Frame, source_year: int) -> Frame: + """``frame`` with each household's ``source_year`` and + ``source_household_key``, before any clone exists. + + The key is written as Python strings in an object column whatever pandas + infers for text: the dtype the SPI support channel's own write gives with + string inference off, as in the legacy plan, so that plan's object-string + columns stay what they were. The graph casts the column to its declared + string dtype. + """ + + household = frame.table("household").copy() + ids = household[UK_NATIONAL_SCHEMA.id_column("household")].tolist() + household[SOURCE_YEAR_COLUMN] = np.full(len(household), source_year, np.int64) + household[SOURCE_HOUSEHOLD_KEY_COLUMN] = pd.Series( + [f"{source_year}:{source}" for source in ids], + index=household.index, + dtype=object, + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = household + return Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def expand_uk_geographic_clones( + frame: Frame, *, clones: int +) -> UKGeographicExpandResult: + """``frame`` as ``clones`` geographic clones of every household, clone-major. + + Clone 0 is the frame's own rows and ids. Clone ``k`` repeats every + person, benefit unit and household with each id minted in the + ``geographic_support`` field at ordinal ``k``. Every row carries its + clone in its table's ``{entity}_clone_index``, every household its + ``source_household_id`` (the clone-0 id), ``source_year`` and + ``source_household_key``, and each clone ``w / clones`` of its + household's design weight, with one mass record stating + :data:`GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON`. At one clone the rows, ids + and weights are the frame's own; only those columns and the record are + added. + + Refused, by count or by name: a clone count outside [1, + :data:`GEOGRAPHIC_CLONE_LIMIT`]; weights on any entity but the household, + or of any kind but design; a frame already carrying a clone index, a + source column or a copy step's column; id or membership columns that are + not int64; a time period that is not a year; incoming ids at or above the + field's radix; and every refusal of the shared core. + """ + + count = _clone_count(clones) + if not isinstance(frame, Frame): + raise TypeError("The geographic expansion clones a Frame.") + _refuse_other_weighted_entities(frame) + validate_uk_national_frame(frame) + _refuse_carried_lineage(frame) + _refuse_narrow_ids(frame) + base = _with_source_year_and_key(frame, _source_year(frame)) + expanded = expand_clones(base, clones=count, declaration=UK_GEOGRAPHIC_CLONES) + validate_uk_national_frame(expanded) + sources = record_source_households( + expanded, "household", stage=GEOGRAPHIC_EXPAND_STAGE_NAME + ) + return UKGeographicExpandResult( + frame=expanded, + clone_count=count, + source_households=distinct_carriers(sources.tolist()), + ) + + +@dataclass(frozen=True) +class UKGeographicExpandStageTransform: + """Whole-stage transform for the geographic expansion (G1). + + ``clones`` is the clone count ``K``. Constructing the transform checks + the stage it runs as and the count; running it calls + :func:`expand_uk_geographic_clones`. + """ + + stage: SourceStageSpec + clones: int + last_result: UKGeographicExpandResult | None = field( + default=None, init=False, repr=False + ) + + def __post_init__(self) -> None: + _assert_geographic_expand_stage(self.stage) + _clone_count(self.clones) + + def __call__(self, frame: Frame) -> Frame: + _assert_geographic_expand_stage(self.stage) + result = expand_uk_geographic_clones(frame, clones=self.clones) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py new file mode 100644 index 0000000000..79fb8f50b5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py @@ -0,0 +1,285 @@ +"""Each geographic clone's geography key, from its source and clone path (microcosm#931). + +G1b of the clone-first plan runs right after the geographic expansion +(:mod:`~microcosm.build.uk_runtime.geographic_expand`). It gives every +household the key the area assignment draws on, ``geography_household_key``, +as the dense line's ``uk.full.identity`` writes it: ``household_draw_key`` +over the FRS source, the household's FRS vintage, its true source and its +copy path. + +- **Path.** The shared reader (``source_lineage.copy_paths`` on + ``household_lineage.UK_COPY_LINEAGE``) reads it from the lineage columns, + never from ids, weights or values. Before any copy step it is ``()`` for + clone 0 and ``(("geographic_support", k),)`` for clone ``k``. +- **Vintage.** The FRS vintage frs_release.json pins for the household's + ``source_year`` (:func:`frs_vintages_by_source_year`). A year it pins no + vintage for is refused. +- **Dtype.** The key is written in the graph's ``string`` dtype (pandas' + ``StringDtype("python")``), as ``uk.full.identity`` writes it, so it is not + a legacy object-string column. + +So clone 0's key is the key the dense identity gives the same base household, +clone ``k``'s the key it gives that household's clone ``k``, and growing K +never changes an existing household's key. + +Refusals give counts, columns, dtypes and declared constants only, never a +key, an id or a year of a row. The evidence gives the distinct source +households rounded through ``receipt_counts`` (a household's clones counted +once), that the keys are unique, the source and the vintages; no key is ever +printed. + +No stage runs this step yet, so it moves no output and no node key. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.source_lineage import copy_paths +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.atomic_area_support import IDENTITY_COLUMN +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.frs_release import UKFRSRelease, load_uk_frs_release +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.geographic_expand import SOURCE_YEAR_COLUMN +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_SUPPORT_BRANCH, + UK_COPY_LINEAGE, +) +from microcosm.build.uk_runtime.national_frame import validate_uk_national_frame +from microcosm.build.uk_runtime.receipt_counts import ( + distinct_carriers, + rounded_count, +) +from microcosm.frame import Frame +from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN +from microcosm.graph.population import dtype_for_token + +__all__ = [ + "GEOGRAPHIC_IDENTITY_INPUTS", + "GEOGRAPHIC_IDENTITY_SOURCE", + "GEOGRAPHIC_IDENTITY_STAGE_NAME", + "UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS", + "UKGeographicIdentityResult", + "UKGeographicIdentityStageTransform", + "frs_vintages_by_source_year", + "key_uk_households_by_clone_path", +] + +GEOGRAPHIC_IDENTITY_STAGE_NAME = "geographic_identity" +#: The source every key names: the dense line's identity source (the default +#: ``identity_source`` of ``graph_population.append_uk_population_nodes``), so +#: clone 0 keys as the dense line keys a base household. +GEOGRAPHIC_IDENTITY_SOURCE = "frs" +#: The household columns the step reads. +GEOGRAPHIC_IDENTITY_INPUTS = ( + SOURCE_HOUSEHOLD_ID_COLUMN, + SOURCE_YEAR_COLUMN, + ladder_clone_index_column("household"), +) +UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS = (IDENTITY_COLUMN,) + + +def frs_vintages_by_source_year( + release: UKFRSRelease | None = None, +) -> Mapping[int, str]: + """The FRS vintage frs_release.json pins for each source year it covers. + + The release pins one vintage, which the loader checks starts with its + survey year. The geographic expansion writes each household's + ``source_year`` from the frame's time period, the release's base year, as + the SPI support channel does. So both years map to the vintage (one year + when they are equal), as the dense identity keys every household with + it; any other year maps to none. ``release`` defaults to the packaged pin. + """ + + release = load_uk_frs_release() if release is None else release + vintage = str(release.vintage) + return MappingProxyType( + {int(release.survey_year): vintage, int(release.base_year): vintage} + ) + + +@dataclass(frozen=True) +class UKGeographicIdentityResult: + """The keyed frame and what the step's evidence reads of it. + + ``source_households`` is the exact count of distinct source households; + the evidence gives it rounded, and neither field prints in the repr. + """ + + frame: Frame = field(repr=False) + source_households: int = field(repr=False) + source_vintages: tuple[str, ...] + + def evidence(self) -> dict[str, object]: + # A result exists only for unique keys: repeated keys are refused. + return { + "stage": GEOGRAPHIC_IDENTITY_STAGE_NAME, + "source_households": rounded_count(self.source_households), + "unique": True, + "source": GEOGRAPHIC_IDENTITY_SOURCE, + "source_vintages": list(self.source_vintages), + "inputs": list(GEOGRAPHIC_IDENTITY_INPUTS), + } + + +def _assert_geographic_identity_stage(stage: SourceStageSpec) -> None: + """The stage a transform runs as: the step's name, household grain, and + exactly the key as its output, with no rewrites.""" + + if not isinstance(stage, SourceStageSpec): + raise TypeError("The geography identity runs as a SourceStageSpec stage.") + if stage.stage != GEOGRAPHIC_IDENTITY_STAGE_NAME: + raise ValueError( + f"Expected stage {GEOGRAPHIC_IDENTITY_STAGE_NAME!r}, got {stage.stage!r}." + ) + if stage.grain != "household": + raise ValueError( + "The geography identity keys households; grain must be 'household', " + f"got {stage.grain!r}." + ) + if tuple(stage.outputs) != UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS or tuple( + stage.rewrites + ): + raise ValueError( + "The geography identity writes exactly its key; the stage must " + f"declare outputs {UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS} and no " + f"rewrites, got {tuple(stage.outputs)} / {tuple(stage.rewrites)}." + ) + + +def _require_int64(household: pd.DataFrame, column: str) -> np.ndarray: + if column not in household.columns: + raise ValueError( + f"The geography identity reads household {column!r}, which the " + "geographic expansion writes; the frame lacks it." + ) + dtype = household[column].dtype + if dtype != np.dtype("int64"): + raise ValueError( + f"The geography identity reads an int64 {column!r}; it holds {dtype}." + ) + return household[column].to_numpy() + + +def _vintages(years: np.ndarray, release: UKFRSRelease | None) -> list[str]: + """Each household's FRS vintage, refusing years the release pins none for.""" + + by_year = frs_vintages_by_source_year(release) + unmapped = int((~np.isin(years, np.fromiter(by_year, np.int64))).sum()) + if unmapped: + raise ValueError( + f"{unmapped} household(s) carry a {SOURCE_YEAR_COLUMN} that " + "frs_release.json pins no FRS vintage for; it pins the source " + f"year(s) {sorted(by_year)} (microcosm#931)." + ) + return [by_year[year] for year in years.tolist()] + + +def key_uk_households_by_clone_path( + frame: Frame, *, release: UKFRSRelease | None = None +) -> UKGeographicIdentityResult: + """``frame`` with every household's ``geography_household_key``. + + Each key is ``household_draw_key`` over :data:`GEOGRAPHIC_IDENTITY_SOURCE`, + the vintage ``release`` (frs_release.json by default) pins for the + household's ``source_year``, its ``source_household_id`` and its copy path + read from the lineage columns, which must hold the geographic expansion's + clone index. Rows, ids, weights, the mass log and the metadata are the + frame's own. + + Refused, by count or by name: a frame that already carries the key, or + lacks the clone index or an int64 ``source_household_id`` or + ``source_year``; malformed lineage, through the shared reader; a source + year the release pins no vintage for; and keys that are not unique. + """ + + if not isinstance(frame, Frame): + raise TypeError("The geography identity keys a Frame.") + validate_uk_national_frame(frame) + household = frame.table("household") + if IDENTITY_COLUMN in household.columns: + raise ValueError( + f"The frame already carries {IDENTITY_COLUMN!r}; the geography identity " + "keys each household once." + ) + _require_int64(household, SOURCE_HOUSEHOLD_ID_COLUMN) + years = _require_int64(household, SOURCE_YEAR_COLUMN) + sources, paths = copy_paths( + household, UK_COPY_LINEAGE, require=(GEOGRAPHIC_SUPPORT_BRANCH,) + ) + vintages = _vintages(years, release) + keys = [ + household_draw_key( + source=GEOGRAPHIC_IDENTITY_SOURCE, + source_vintage=vintage, + source_household_id=source, + clone_path=path, + ) + for source, vintage, path in zip(sources.tolist(), vintages, paths, strict=True) + ] + repeated = len(keys) - len(set(keys)) + if repeated: + raise ValueError( + f"{repeated} household(s) repeat an earlier household's geography key; " + "every household needs its own source and clone path (microcosm#931)." + ) + keyed = household.copy() + keyed[IDENTITY_COLUMN] = pd.Series( + keys, index=household.index, dtype=dtype_for_token("string") + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = keyed + result = Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + validate_uk_national_frame(result) + return UKGeographicIdentityResult( + frame=result, + source_households=distinct_carriers(sources.tolist()), + source_vintages=tuple(sorted(set(vintages))), + ) + + +@dataclass(frozen=True) +class UKGeographicIdentityStageTransform: + """Whole-stage transform for the geography identity (G1b). + + Constructing it checks the stage it runs as; running it calls + :func:`key_uk_households_by_clone_path` with the packaged + frs_release.json. + """ + + stage: SourceStageSpec + last_result: UKGeographicIdentityResult | None = field( + default=None, init=False, repr=False + ) + + def __post_init__(self) -> None: + _assert_geographic_identity_stage(self.stage) + + def __call__(self, frame: Frame) -> Frame: + _assert_geographic_identity_stage(self.stage) + result = key_uk_households_by_clone_path(frame) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index c0b642e848..7223741120 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -97,6 +97,7 @@ frs_spine, frs_take_up, gambling_participation, + geographic_clones, graph_frames, hmrc_capital_gains, hmrc_company_car_statistics, @@ -429,8 +430,9 @@ # and income proxy from cgt_imputation (whose loss relief evaluates a named # term), and weigh their copies through spi_support's exact-total # correction; the support split also takes the adult age and oldest-adult - # rule from cgt_structure, and the residential split its logistic from - # cgt_asset_type; the incidence clone sums cgt_imputation's income proxy + # rule from cgt_structure and reads the frame's geographic clone count + # through geographic_clones, and the residential split takes its logistic + # from cgt_asset_type; the incidence clone sums cgt_imputation's income proxy # components into the income behind its propensity; the gains spine names # HMRC's regional conditioning rows by microcosm.calibrate's UK region tier, # and it and the asset types key each walk's offset on its label through @@ -440,6 +442,7 @@ *_FEED_FACT_MODULES, cgt_imputation, cgt_structure, + geographic_clones, hmrc_capital_gains, hmrc_income, spi_support, @@ -844,10 +847,17 @@ # The receipts' counting rules and the shared lineage readers: every floor # counts true source households through receipt_counts, and the FRS-trained # fits fold geographic clones through household_lineage, both reading -# source_lineage on household_lineage's declaration (microcosm#931). An edit -# to any of the three moves the hash of every stage whose module or helpers -# import one of them; test_uk_graph derives this set from the imports. -_COUNTING_MODULES = (receipt_counts, household_lineage, source_lineage) +# source_lineage on household_lineage's declaration, which names the +# geographic clone index through geographic_clones (microcosm#931). An edit +# to any of the four moves the hash of every stage whose module or helpers +# import one of the first three; test_uk_graph derives this set from the +# imports. +_COUNTING_MODULES = ( + receipt_counts, + household_lineage, + source_lineage, + geographic_clones, +) _COUNTING_STAGES = frozenset( { "frs_spine", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index 106fcedd27..ffdd61e97f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -50,6 +50,7 @@ atomic_area_support, atomic_household_identity, brma_area_layer, + geographic_clones, geography_ladder, local_authority_input, national_frame, @@ -61,6 +62,7 @@ rowwise_geography, spi_support, ) +from .geographic_clones import ladder_clone_index_column from .geography_ladder import ( UK_GEOGRAPHY_LADDER_COLUMNS, derive_uk_ladder_locations, @@ -82,7 +84,7 @@ ) from .national_frame import UK_NATIONAL_SCHEMA from .receipt_counts import rounded_count, source_household_ids -from .rowwise_dataset import expand_uk_geographic_pool, ladder_clone_index_column +from .rowwise_dataset import expand_uk_geographic_pool POPULATION_RECEIPT_TYPE = ArtifactType("microcosm.uk.population-receipt", 1) LOCATION_DRAW_TYPE = ArtifactType("microcosm.uk.ladder-location-draw", 1) @@ -207,17 +209,19 @@ def implementation_hash(self) -> str: # The dense population nodes also compute through what these call: # the sample's household draw and mass normalisation (frame_sampling), # its strata columns (spi_support's replacement strata), the clones' - # minted ids and copied tables (rowwise_geography), the national frame - # they rebuild and the engine adapter's contracts it stamps on them, - # the local authority names the geography resolves, the gate's - # results, their artifacts' canonical encoding and the normalisation's - # weight-update receipt (microcosm#931); test_uk_graph derives the set - # from what the kernels name. + # minted ids and copied tables (rowwise_geography), the clone index + # columns' names (geographic_clones), the national frame they rebuild + # and the engine adapter's contracts it stamps on them, the local + # authority names the geography resolves, the gate's results, their + # artifacts' canonical encoding and the normalisation's weight-update + # receipt (microcosm#931); test_uk_graph derives the set from what the + # kernels name. return source_hash( type(self), geography_ladder, national_sampling, rowwise_dataset, + geographic_clones, graph_evidence, frame_sampling, gates, @@ -445,14 +449,16 @@ def implementation_hash(self) -> str: from .. import source_lineage from . import household_lineage - # The clone paths are read through the lineage readers, whose default - # source column frame_concepts names, and the draw keys are encoded - # canonically (microcosm#931). + # The clone paths are read through the lineage readers, on the clone + # index column geographic_clones names and the default source column + # frame_concepts names, and the draw keys are encoded canonically + # (microcosm#931). return source_hash( type(self), atomic_household_identity, atomic_area_support, rowwise_dataset, + geographic_clones, household_lineage, source_lineage, frame_concepts, @@ -517,7 +523,7 @@ def implementation_hash(self) -> str: from . import household_lineage, tenure_constants # The spine keys come through the lineage readers and the clone index - # column rowwise_dataset names, the owner mask through + # column geographic_clones names, the owner mask through # tenure_constants' tenure categories and the owner uniforms through # keyed draws; the spine keys and the quantile artifact are encoded # canonically (microcosm#931). @@ -529,6 +535,7 @@ def implementation_hash(self) -> str: source_lineage, frame_concepts, rowwise_dataset, + geographic_clones, tenure_constants, keyed_uniform, canonical_json, @@ -871,7 +878,7 @@ def implementation_hash(self) -> str: from . import household_lineage, receipt_counts # The spine keys come through the lineage readers and the clone index - # column rowwise_dataset names, and are encoded canonically; the + # column geographic_clones names, and are encoded canonically; the # tie-breaks are keyed draws; the receipt counts source households on # national_frame's schema and rounds them in receipt_counts # (microcosm#931). @@ -884,6 +891,7 @@ def implementation_hash(self) -> str: source_lineage, frame_concepts, rowwise_dataset, + geographic_clones, national_frame, receipt_counts, keyed_uniform, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index dc256e0efa..1548687246 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -53,6 +53,7 @@ from . import geography_ladder, national_frame, weight_year from .atomic_area_support import uk_area_code_frames, without_uk_native_alias_columns from .dataset_size import UK_SIZE_L2_PARAM_KEYS +from .geographic_clones import ARTIFACT_CLONE_INDEX_COLUMN, ladder_clone_index_column from .geography_ladder import ( UK_EXPORT_AREA_CODE_COLUMNS, export_area_code_columns, @@ -77,11 +78,7 @@ uk_time_period, validate_uk_national_frame, ) -from .rowwise_dataset import ( - ARTIFACT_CLONE_INDEX_COLUMN, - ladder_clone_index_column, - load_uk_rowwise_dataset, -) +from .rowwise_dataset import load_uk_rowwise_dataset from .weight_year import uk_dataset_year_frame, uk_weight_projection_factor EXPORT_DESCRIPTOR_TYPE = ArtifactType("microcosm.uk.full-export-descriptor", 1) @@ -362,6 +359,7 @@ def implementation_hash(self) -> str: from .. import gates from . import ( atomic_area_support, + geographic_clones, graph_population, hmrc_uprating, local_authority_input, @@ -372,7 +370,7 @@ def implementation_hash(self) -> str: # The descriptor also computes through the ladder gate's results and # region codes (gates, rowwise_geography, local_authority_input), the # area code frames (atomic_area_support), the engine tables - # (frame_materialize) and the clone index columns rowwise_dataset + # (frame_materialize) and the clone index columns geographic_clones # names, the engine's weight projection factor (hmrc_uprating's # pins), the frame it rebuilds (graph_population) and the canonical # encoding of the descriptor (microcosm#931). The gate report it reads @@ -388,6 +386,7 @@ def implementation_hash(self) -> str: hmrc_uprating, local_authority_input, rowwise_dataset, + geographic_clones, rowwise_geography, frame_materialize, canonical_json, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py index e3f38a4da1..d8d62f33a5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py @@ -26,8 +26,8 @@ fold_replicas, replica_keys, ) +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.national_frame import UK_NATIONAL_SCHEMA -from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column __all__ = [ "GEOGRAPHIC_SUPPORT_BRANCH", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py index 897fe4db92..68b39c7c12 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py @@ -44,6 +44,7 @@ def implementation_hash(self) -> str: from .. import gates from . import ( atomic_area_support, + geographic_clones, geography_ladder, graph_population, hmrc_uprating, @@ -56,14 +57,16 @@ def implementation_hash(self) -> str: # The description computes through what the export kernel's does # (microcosm#931): the ladder gate's results and region codes, the - # area code frames, the engine tables and clone index columns, the - # dataset year's weights and the engine's projection factor, the - # frame it rebuilds, and the descriptor's canonical encoding. + # area code frames, the engine tables and the clone index columns + # geographic_clones names, the dataset year's weights and the + # engine's projection factor, the frame it rebuilds, and the + # descriptor's canonical encoding. return source_hash( sys.modules[__name__], graph_terminal, gates, atomic_area_support, + geographic_clones, geography_ladder, graph_population, hmrc_uprating, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index 177b867513..b7ea59e5ad 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -17,6 +17,16 @@ import pandas as pd from microcosm.build.gates import GateResult + +# Moved to geographic_clones (microcosm#931): the spine stages that count by +# lineage read the clone index columns' names through household_lineage and +# hash that small module, not this one, and the full build's population and +# export kernels hash it beside this one. Kept importable from here for this +# module's other callers. +from microcosm.build.uk_runtime.geographic_clones import ( + ARTIFACT_CLONE_INDEX_COLUMN, + ladder_clone_index_column, +) from microcosm.build.uk_runtime.geography_ladder import ( UK_GEOGRAPHY_LADDER_COLUMNS, UkOaLadder, @@ -84,10 +94,6 @@ HOUSEHOLD_ID_COLUMNS = ("household_id",) UK_SINGLE_YEAR_TABLES = ("person", "benunit", "household", "time_period") -#: Clone-index column name in the written artifact (one name per table — -#: the legacy single-year schema, unchanged by the Frame carrier). -ARTIFACT_CLONE_INDEX_COLUMN = "clone_index" - _INT64_MAX = int(np.iinfo(np.int64).max) @@ -130,17 +136,6 @@ def refuse_dense_clone_id_overflow( ) -def ladder_clone_index_column(entity: str) -> str: - """The in-memory, per-entity clone-index column on the ladder frame. - - Frame enforces globally unique column names across entity tables, so the - artifact's shared ``clone_index`` name cannot appear on all three tables - in memory; the writer renames these back at the export boundary. - """ - - return f"{entity}_{ARTIFACT_CLONE_INDEX_COLUMN}" - - def _refuse_preassigned_geography( table: pd.DataFrame, *, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py index 11986721a5..d29f17f367 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py @@ -30,6 +30,7 @@ _assert_closed_world_operations, ) from microcosm.build.uk_runtime.cgt_support import HOUSEHOLD_IS_CGT_SUPPORT_COPY +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.household_lineage import collapse_geographic_clones from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, @@ -343,14 +344,28 @@ def draw_spi_bonuses( source_ids: np.ndarray, donors: np.ndarray, spi: np.ndarray, + clone_indices: np.ndarray | None = None, ) -> UKSpiBonusDraw: """Each SPI-channel adult's bonus, as a share of pay drawn from FRS adults. - ``donors`` marks the base-FRS adults (no clone, no support copy) and - ``spi`` the SPI-channel adults. Each SPI adult with pay takes the bonus - share of pay of one donor with pay in its pay cell, drawn by person weight - on a uniform keyed on its own ``person_id`` (``BONUS_SACRIFICE_SPI_DRAW``); - an SPI adult without pay has no bonus. Every other row keeps its own. + ``donors`` marks the base-FRS adults (no capital-gains clone, no support + copy) and ``spi`` the SPI-channel adults. Each SPI adult with pay takes + the bonus share of pay of one donor with pay in its pay cell, drawn by + person weight on a uniform keyed on its own ``person_id`` + (``BONUS_SACRIFICE_SPI_DRAW``); an SPI adult without pay has no bonus. + Every other row keeps its own. + + ``clone_indices`` gives each row's geographic clone (microcosm#931); None, + as on every spine today, puts every row in clone 0. The pay cells and the + donor-source floor pool every clone's donors, the floor counted in + distinct ``source_ids``. Each SPI adult draws from the donors of its own + clone in its cell, in id order, on the same keyed uniform. So a frame of + one clone draws as before, value for value. Where the clones agree on the + donors' pay, clone 0 of ``K``, whose ids are the one-clone ids, draws on + the one-clone pool at ``w / K``, up to the 1/K split's rounding, which can + move a pooled cell edge where a quantile falls exactly on a cumulative + weight share. A cell with no donor in an SPI adult's clone is refused, + without a count. """ bonus = np.maximum(np.asarray(bonus, dtype=float), 0.0) @@ -404,21 +419,51 @@ def donor_sources(group: list[int]) -> int: salt=BONUS_SACRIFICE_SPI_DRAW_SALT, ) ids = np.asarray(person_ids) + clones = ( + np.zeros(len(pay), dtype=np.int64) + if clone_indices is None + else np.asarray(clone_indices) + ) for cell in np.unique(cells[recipients]): - pool = np.flatnonzero(paid_donors & (cells == cell)) - if not len(pool): - raise ValueError("An SPI-channel bonus pay cell has no donor to draw.") - pool = pool[np.argsort(ids[pool], kind="stable")] - pool_cumulative = np.cumsum(weights[pool]) / weights[pool].sum() - takers = np.flatnonzero(recipients & (cells == cell)) - picks = np.minimum( - np.searchsorted(pool_cumulative, uniforms[takers], side="right"), - len(pool) - 1, - ) - drawn[takers] = share[pool[picks]] * pay[takers] + in_cell = cells == cell + # The cell's SPI adults in each clone draw from that clone's donors + # alone; with one clone, from the whole cell. + for clone in np.unique(clones[recipients & in_cell]): + members = in_cell & (clones == clone) + pool = np.flatnonzero(paid_donors & members) + if not len(pool): + raise ValueError( + "An SPI-channel bonus pay cell has no donor to draw in an " + "SPI adult's geographic clone." + ) + pool = pool[np.argsort(ids[pool], kind="stable")] + pool_cumulative = np.cumsum(weights[pool]) / weights[pool].sum() + takers = np.flatnonzero(recipients & members) + picks = np.minimum( + np.searchsorted(pool_cumulative, uniforms[takers], side="right"), + len(pool) - 1, + ) + drawn[takers] = share[pool[picks]] * pay[takers] return UKSpiBonusDraw(bonus=drawn, recipients=recipients, cells=len(groups)) +def _geographic_clone_indices( + person_households: pd.Series, households: pd.DataFrame +) -> np.ndarray | None: + """Each person's geographic clone: its household's ``household_clone_index``. + + ``households`` is indexed by ``household_id``. None without the column, + which is one clone, as every spine is today (microcosm#931). The stage's + geographic fold reads the column first, as the ``geographic_support`` + branch's ordinals, and refuses one that is not a nonnegative integer. + """ + + column = ladder_clone_index_column("household") + if column not in households.columns: + return None + return person_households.map(households[column]).to_numpy(dtype=np.int64) + + @dataclass(frozen=True) class UKSalarySacrificeResult: """Transformed frame and the full headcount executed-effect receipt.""" @@ -753,6 +798,8 @@ def impute_salary_sacrifice( "person", stage="The salary-sacrifice stage", ) + # Each SPI adult draws a bonus share from its own geographic clone's + # donors: its household's clone index, or clone 0 without the column. spi_draw = draw_spi_bonuses( bonus=bonus_source, employment_income=employment_income, @@ -763,6 +810,7 @@ def impute_salary_sacrifice( channels.eq("frs") & ~clones.astype(bool) & ~support_copies.astype(bool) ).to_numpy(), spi=channels.ne("frs").to_numpy(), + clone_indices=_geographic_clone_indices(person_households, households), ) bonus = convert_bonuses_to_bonus_sacrifice( bonus=spi_draw.bonus, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py index 30f2455d01..7eef85f911 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py @@ -60,7 +60,7 @@ #: (:func:`~microcosm.build.stochastic_assignment.binomial_share_tolerance`). SPI_SHARE_GATE_RULE = "binomial_share_band" #: The geographic clone index the clone-first spine gives every household -#: (``rowwise_dataset.ladder_clone_index_column("household")``, the +#: (``geographic_clones.ladder_clone_index_column("household")``, the #: ``geographic_support`` branch of the UK copy lineage; microcosm#931). Where #: the frame carries it, the prior allocation's strata split by it, so each #: clone keeps its own mass. It is spelled here, not imported, so a change to diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_capital_coherence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_capital_coherence.py index 07f8d44e08..cb10e54aba 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_capital_coherence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_capital_coherence.py @@ -24,7 +24,7 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.uk_runtime.frs_spine import UC_CAPITAL_UNAVAILABLE -from microcosm.build.uk_runtime.household_lineage import collapse_geographic_clones +from microcosm.build.uk_runtime.household_lineage import fold_geographic_clones from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, @@ -65,12 +65,13 @@ #: GBP 6,000, 16,000, 50,000 and 200,000 of capital yield at 4%, then above. UC_CAPITAL_INCOME_BAND_EDGES = (0.0, 240.0, 640.0, 2_000.0, 8_000.0) UC_CAPITAL_IMPLIED_YIELD = 0.04 -#: The donors a cell needs before it stops widening. It counts the cell's -#: donor benefit-unit rows after the geographic fold, so a high-value -#: support copy counts as a donor of its own. It is a row threshold, not a -#: disclosure floor. Whether it should count true sources is for the -#: geography-first move's audit of K-sensitive constants (microcosm#931), -#: since switching it can move outputs. +#: The donors a cell needs before it stops widening, counted in survey +#: records (María's ruling of 2026-11-10, microcosm#931): a benefit unit's +#: geographic clones count once, whether they fold into one row or differ and +#: donate apart, and every other copy, a high-value support copy say, counts +#: as a record of its own. A frame without geographic clones holds one +#: record per row, so there the count is the cell's donor rows, as before. +#: It is a donor threshold, not a disclosure floor. UC_CAPITAL_MINIMUM_CELL_DONORS = 20 #: The declared order in which a cell short of donors widens; reporter status #: is never coarsened. @@ -651,11 +652,14 @@ def _redraw_spi_capital( dependent-children band. A cell with fewer than ``minimum_cell_donors`` donors widens in the declared ``UC_CAPITAL_COARSENING`` order; reporter status is never coarsened, and an empty reporter-status pool refuses. - Each survey benefit unit donates once, its geographic clones that agree - on its cell and capital folded into its first row with their weight, so - a cell's donor count does not grow with identical clones - (``household_lineage.collapse_geographic_clones``, microcosm#931). - Returns the number of redrawn units drawn at each level. + Each survey benefit unit donates once: its geographic clones that agree + on its cell and capital fold into its lowest-id clone with their weight, + and clones that differ donate apart, each variant on its lowest-id clone + with its clones' weight (``household_lineage.fold_geographic_clones``, + microcosm#931). A cell's donors are counted in survey records, the + distinct replica keys among its rows, so neither identical nor differing + clones grow the count, while every other copy counts as a record of its + own. Returns the number of redrawn units drawn at each level. """ if not isinstance(minimum_cell_donors, int) or minimum_cell_donors <= 0: @@ -665,9 +669,10 @@ def _redraw_spi_capital( income_band = _investment_income_band(investment) last_band = len(UC_CAPITAL_INCOME_BAND_EDGES) # Domain-validated upstream: every non-sentinel value is >= 0. A donor's - # clones fold where they agree on its cell and the capital it donates; - # clones that differ donate apart, each with its clones' weight. - donor, donor_weight = collapse_geographic_clones( + # clones fold onto its lowest-id clone where they agree on its cell and + # the capital it donates; clones that differ donate apart, each variant + # with its clones' weight. Every clone of a donor shares its key group. + fold = fold_geographic_clones( {"person": person, "benunit": benunit, "household": household}, "benunit", base & (capital >= 0.0) & (weights > 0.0), @@ -682,6 +687,7 @@ def _redraw_spi_capital( } ), ) + donor, donor_weight = fold.kept, fold.weights target_ids = benunit["benunit_id"].to_numpy() draws = stable_identity_uniforms( target_ids, @@ -736,7 +742,7 @@ def _redraw_spi_capital( ( (name, pool) for name, pool in candidates - if int(pool.sum()) >= minimum_cell_donors + if _survey_records(fold.key_group, pool) >= minimum_cell_donors ), candidates[-1], ) @@ -761,6 +767,17 @@ def _redraw_spi_capital( return levels +def _survey_records(key_group: np.ndarray, pool: np.ndarray) -> int: + """The survey records among a donor pool's rows: its distinct key groups. + + ``key_group`` is the geographic fold's code of each row's replica key, so + a record's clones count once whether they fold into one row or donate + apart; a frame without geographic clones holds one record per row. + """ + + return int(np.unique(key_group[pool]).size) + + def _investment_income_band(investment: np.ndarray) -> np.ndarray: """Band 0 for no investment income, then one band per declared edge.""" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 39e133b367..57cc32d87c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -71,6 +71,10 @@ from microcosm.frame import WeightKind from test_support.microcosm_build.uk_cgt_support import ( BAND_INCOMES, + NESTING_INCOMES, + NESTING_TOP_CELLS, + NESTING_WEALTH, + NESTING_WEIGHTS, PARAMETERS, drift, support_distribution, @@ -774,11 +778,11 @@ def _scenario_split_at(clone_count): def test_the_clone_index_column_is_the_lineages_geographic_branch(): from microcosm.build.uk_runtime.cgt_support import _GEOGRAPHIC_CLONE_INDEX_COLUMN + from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.household_lineage import ( GEOGRAPHIC_SUPPORT_BRANCH, UK_COPY_LINEAGE, ) - from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column (branch,) = [ branch @@ -793,6 +797,39 @@ def test_the_clone_index_column_is_the_lineages_geographic_branch(): ) +def test_every_spelling_of_the_household_clone_index_is_the_leafs_name(): + """geographic_clones names the clone index columns and counts the clones + (microcosm#931). The household's name is also spelled out in this stage's + module, in the SPI support channel's, whose stages key on it there, and as + microcosm-frame's engine batch column, whose package does not depend on + microcosm-build. Each spelling equals the leaf's name, and rowwise_dataset + and this stage's module re-export the leaf's own definitions.""" + from microcosm.build.uk_runtime import ( + geographic_clones, + rowwise_dataset, + spi_support, + ) + from microcosm.frame.adapters.policyengine_uk_population import ( + UK_SPINE_ENGINE_BATCH_COLUMN, + ) + + name = geographic_clones.ladder_clone_index_column("household") + assert name == f"household_{geographic_clones.ARTIFACT_CLONE_INDEX_COLUMN}" + assert ( + name + == cgt_support._GEOGRAPHIC_CLONE_INDEX_COLUMN + == spi_support.SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN + == UK_SPINE_ENGINE_BATCH_COLUMN + == "household_clone_index" + ) + for module, names in ( + (rowwise_dataset, ("ARTIFACT_CLONE_INDEX_COLUMN", "ladder_clone_index_column")), + (cgt_support, ("geographic_clone_count",)), + ): + for attribute in names: + assert getattr(module, attribute) is getattr(geographic_clones, attribute) + + def test_geographic_clone_count_is_one_more_than_the_largest_index(): assert geographic_clone_count(pd.DataFrame({"household_id": [1, 2]})) == 1 for indices, expected in (([0, 0], 1), ([0, 1, 2, 1, 0], 3), ([2, 0, 1], 3)): @@ -1860,22 +1897,11 @@ def _leaves(value): # clone, the one exception copy_rule names. That copy is then off w / n by # about an ulp of the frame's total weight (thousands of ulps of its own, a # relative error below 1e-12), within the weights' nesting tolerance and the -# gate's; at one, three, 11 and 15 clones no correction is needed. - -NESTING_WEIGHTS = [ - 900.0, - 1_800.0, - 4_020.0, - 7_620.0, - float(np.nextafter(300.0, 0.0)), - 40.0, - 1_234.5, - 9_000.0, - 0.0, -] -NESTING_WEALTH = [7e3, 6e3, 5e3, 4e3, 3e3, 2e3, 1e3, 1.0, 1e6] -NESTING_INCOMES = [BAND_INCOMES[0]] * 7 + [BAND_INCOMES[37_700], BAND_INCOMES[0]] -NESTING_TOP_CELLS = {0: {250_000: 1_000_000.0}} +# gate's; at one, three, 11 and 15 clones no correction is needed. The +# scenario's data (NESTING_WEIGHTS, NESTING_WEALTH, NESTING_INCOMES and +# NESTING_TOP_CELLS) are test_support's, shared with +# test_uk_clone_first_nesting.py, which runs it through G1. + NESTING_COPIES = [15, 30, 67, 127, 5, 1, 21, 1, 1] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_expansion_core.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_expansion_core.py index 818c1bf883..f0d0af1aef 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_expansion_core.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_expansion_core.py @@ -1,8 +1,9 @@ """The shared clone expansion core under the UK's declarations (microcosm#931). -With the copy-id table's ``geographic_support`` field, the ladder's -clone-index columns and household design weights, the core must give what the -clone-first stand-in gives (``clone_first_frame`` in +Under G1's declaration (``geographic_expand.UK_GEOGRAPHIC_CLONES``: the +copy-id table's ``geographic_support`` field, the ladder's clone-index columns +and household design weights), the core must give what the clone-first +stand-in gives (``clone_first_frame`` in test_support/microcosm_build/uk_clone_first_frames.py, written without the core): bit for bit on the five id columns, the household clone index, the source household id and the weights, at K = 1, 2, 3, 7 and 15. Invented @@ -15,19 +16,19 @@ import pandas as pd import pytest -from microcosm.build.clone_expansion import ( - CloneDeclaration, - CloneIdRule, - expand_clones, -) +from microcosm.build.clone_expansion import expand_clones from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.geographic_expand import ( + GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON, + UK_GEOGRAPHIC_CLONES, +) from microcosm.build.uk_runtime.national_frame import ( UK_NATIONAL_SCHEMA, uk_national_frame, uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column from microcosm.frame import Frame, WeightKind, Weights from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN from test_support.microcosm_build.uk_clone_first_frames import ( @@ -35,32 +36,9 @@ with_last_household_moved, ) +#: The copy-id field the stand-in mints in, stated here apart from G1's +#: declaration, so the stand-in comparison checks G1 mints in it. _FIELD = "geographic_support" -_REASON = "Geographic clones split each household's weight equally; mass is conserved." - - -def _mint_geographic_clone(ids: np.ndarray, k: int) -> np.ndarray: - return copy_id_table.mint(ids, _FIELD, k) - - -def _require_room_for_geographic_clones(ids: np.ndarray, what: str) -> None: - copy_id_table.require_below(ids, _FIELD, what=what) - - -UK_GEOGRAPHIC_CLONES = CloneDeclaration( - ids=CloneIdRule( - bound=copy_id_table.ordinal_bound(_FIELD), - mint=_mint_geographic_clone, - require_below=_require_room_for_geographic_clones, - ), - clone_index_columns={ - entity: ladder_clone_index_column(entity) - for entity in UK_NATIONAL_SCHEMA.entities - }, - weight_kinds=frozenset({WeightKind.DESIGN}), - reason=_REASON, - source_id_columns={"household": SOURCE_HOUSEHOLD_ID_COLUMN}, -) _ID_COLUMNS = { "person": ("person_id", "person_household_id", "person_benunit_id"), "benunit": ("benunit_id",), @@ -197,7 +175,9 @@ def test_the_mass_record_is_the_same_at_every_clone_count() -> None: (expanded.mass_log[-1].entity, expanded.mass_log[-1].declared_factor) for expanded in records } == {("household", 1.0)} - assert {expanded.mass_log[-1].reason for expanded in records} == {_REASON} + assert {expanded.mass_log[-1].reason for expanded in records} == { + GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON + } def test_a_python_string_column_stays_python_strings_in_every_clone() -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py new file mode 100644 index 0000000000..32c03a8b32 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py @@ -0,0 +1,1072 @@ +"""The clone-first spine nests, stage by stage (microcosm#931, O10). + +Invented households go through the real geographic expansion (G1, +``geographic_expand``), then the SPI support channel (stage 16), the SPI income +band donors (stage 17) and the CGT support split (stage 42), each run by its +stage transform under the packaged stage declaration. G1's stage is a stand-in +until the spine declares it. The frame runs at K = 1, 2, 3, 7 and 15 +geographic clones; it is small, so K = 15 takes well under a second. Each K is +held to K = 1: + +- **Reported first:** stage 17's mass scale at K = 1 and at each K. Clone 0's + stage-17 weights nest on K = 1's only while the scale is the one a one-clone + spine takes. On this frame it is one at every K, and the test asserts it. +- **Exact:** + - clone 0 of G1's output is the one-clone frame, ids and G1's columns + included; + - stage 16 copies, and stage 17 seats, K = 1's households in clone 0; + - every table of every frame carries its clone index with exactly K values, + and every copy carries its parent's. +- **K x each clone-0 weight against K = 1's**, at G1, stage 16 and stage 17: + - Rows an exact-total correction moved, at K or at K = 1, are within + ``rows x K x 2.2e-16`` of K = 1's, relative. ``rows`` counts the one-clone + frame's households at that stage. + - At stage 17, the other incumbents of a funding stratum whose stage-16 + mass holds a moved row are within one ulp per weight-setting step on + their two paths (:data:`_STEP_ULPS`) at every K: the stratum's funding + factor reads that mass, so they need not be bitwise at K = 2. + - Every other row is bitwise at K = 2, a power of two, which scales every + operation exactly. + - At K = 3, 7 and 15, every other row is within one ulp where the 1/K split + is its weight's only rounding. Elsewhere it is within one ulp per + weight-setting step on its two paths (:data:`_STEP_ULPS`). +- **Stage 42, approximately only:** + - The walk can stop inside a household's clones. + - Ties are keyed per row (since b8bd45336), so clone 0 need not win them. + - The K-ulp snap can count one copy fewer. + - Rule B's exact-total correction moves at most one row, by the corrected + rows' tolerance. On the CGT nesting scenario it lands on a copy at 7 and 13 + clones, by less than 1e-12 (the last test, through G1). + - What does nest is each household's family mass and the divided band's + copy counts. +- **The stand-in:** G1's output is ``clone_first_frame``'s on the stand-in's + columns, ids and weights, so the tests written on the stand-in hold for G1. + +Invented values only. Assertion messages give K, stages, counts and booleans, +never a weight or an id. +""" + +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from fractions import Fraction +from itertools import pairwise + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import cgt_support, spi_band_donors, spi_support +from microcosm.build.uk_runtime.cgt_imputation import ( + UK_CGT_INVESTABLE_WEALTH_COLUMNS, + UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, +) +from microcosm.build.uk_runtime.cgt_support import ( + CGT_SUPPORT_COPIES_COLUMN, + CGT_SUPPORT_COPY_INDEX_COLUMN, + CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + CGT_SUPPORT_SPLIT_STAGE_NAME, + HOUSEHOLD_IS_CGT_SUPPORT_COPY, + UKCGTSupportSplitStageTransform, + cgt_support_income_band, +) +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.geographic_expand import ( + GEOGRAPHIC_EXPAND_STAGE_NAME, + UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, + UKGeographicExpandStageTransform, + expand_uk_geographic_clones, +) +from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + uk_national_frame, +) +from microcosm.build.uk_runtime.spi_band_donors import ( + HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, + PERSON_IS_SPI_INCOME_BAND_CARRIER, + SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN, + UK_SPI_INCOME_BAND_DONORS_STAGE_NAME, + UKSPIIncomeBandDonorStageTransform, +) +from microcosm.build.uk_runtime.spi_income import ( + SPI_DONOR_REQUIRED_COLUMNS, + SPI_RECIPIENT_ROLE_COLUMN, + SPIDonorAgeModel, +) +from microcosm.build.uk_runtime.spi_spine import UKSPISupportChannelStageTransform +from microcosm.build.uk_runtime.spi_support import ( + BASE_FRS_SUPPORT_CHANNEL, + SPI_SYNTHETIC_SUPPORT_CHANNEL, + UK_SPI_SUPPORT_STAGE_NAME, + support_channel_column, + support_clone_index_column, + support_source_id_column, +) +from microcosm.frame import Frame, WeightKind +from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN +from test_support.microcosm_build.uk_cgt_support import ( + BAND_INCOMES, + NESTING_INCOMES, + NESTING_TOP_CELLS, + NESTING_WEALTH, + NESTING_WEIGHTS, + PARAMETERS, + support_distribution, + support_frame, +) +from test_support.microcosm_build.uk_clone_first_frames import ( + base_id_tables, + clone_first_frame, +) + +G1 = GEOGRAPHIC_EXPAND_STAGE_NAME +SPI = UK_SPI_SUPPORT_STAGE_NAME +DONORS = UK_SPI_INCOME_BAND_DONORS_STAGE_NAME +CGT = CGT_SUPPORT_SPLIT_STAGE_NAME +#: The stages each frame goes through, in spine order. +_STAGES = (G1, SPI, DONORS, CGT) +#: The stages whose clone-0 weights are held to K = 1's row by row. +_WEIGHED_STAGES = (G1, SPI, DONORS) +#: The clone counts held to one clone: the plan's 2, 3 and 7, and a K = 15 on +#: the same small frame. +_CLONE_COUNTS = (2, 3, 7, 15) +#: Machine epsilon, the 2.2e-16 of the corrected rows' tolerance. +_EPSILON = float(np.finfo(np.float64).eps) +#: The ulps an uncorrected row may move at a K that is not a power of two, +#: where something besides the 1/K split rounds its weight: one per +#: weight-setting step on its two paths. At K, a row passes through the split +#: and each stage up to this one; at one clone, through those stages. So a +#: stage-16 row passes 2 + 1 steps and a stage-17 row 3 + 2. One ulp holds +#: only where the split alone rounds (:func:`_split_only`). Elsewhere the +#: rounding repeats at K and at one clone: stage 16's pension-age share (0.8 +#: kept, 0.2 moved) and its strata's mass sums, and stage 17's funding +#: factors, each computed from a stratum's summed mass. Stage 17's budget +#: also holds, at every K, the incumbents of a stratum whose mass holds a +#: corrected row (:func:`_in_a_reached_stratum`). +_STEP_ULPS = {G1: 1, SPI: 3, DONORS: 5} + +# --- the invented frame ----------------------------------------------------------- + +_HOUSEHOLDS = 90 +_PERIOD = "2024" +#: Invented design weights' seed. The test asserts the premise this seed +#: gives, that stage 16's and stage 17's exact-total corrections both move a +#: row of the one-clone frame, so the corrected rows' class is exercised. +_WEIGHT_SEED = 4 +#: Given to stage 16, so no engine is needed. +_STATE_PENSION_AGE = 66 +#: Stage 16's declared count times this is the share's expected copies: half +#: of the frame's households, so every clone's region and pension-age strata +#: keep copies at K = 15. +_SAMPLE_FRACTION = 0.0045 +#: The invented tape's two regions. +_REGIONS = ("SOUTH_EAST", "LONDON") +#: The Table 3 income band of each household's oldest adult, by its pay. +_CGT_BANDS = (0, 37_700, 50_000) +#: Invented taxpayers per reserved band (GBP 200,000 and up). +_BAND_TAXPAYERS = { + 200_000: 300_000.0, + 500_000: 60_000.0, + 1_000_000: 20_000.0, + 2_000_000: 10_000.0, +} +#: The invented Table 3 joint asks band 0 for far more than its pool, so its +#: whole pool is divided at every K; asks band 37,700 for 2 x 1.25 x 1,000 = +#: 2,500, which the walk crosses partway; and asks band 50,000 for nothing. +_EXHAUSTED_BAND = 0 +_CROSSING_BAND = 37_700 +_CGT_TOP_CELLS = {_EXHAUSTED_BAND: {250_000: 1e9}, _CROSSING_BAND: {250_000: 1_000.0}} +#: The invented tape's pay at each of its six total-income levels: two below +#: GBP 200,000, then one in each reserved band. +_TAPE_PAY = (30_000.0, 80_000.0, 260_000.0, 620_000.0, 1_300_000.0, 2_600_000.0) + + +def _base_frame() -> Frame: + """Invented FRS-shaped households with what stages 16, 17 and 42 read, and + no lineage. + + - **Ids** are ``base_id_tables``': sernums 1 to 90, with one to four people + in one or two benefit units. + - **Members.** The first member is the oldest adult. In every third + household they are 70, so stage 16 has pension-age strata in both + regions. Elsewhere they are 45 to 54, the tape's age band, so stage 17 + has candidates in every clone. The second member is a 44-year-old adult, + the rest are children, and the first two are claimants or partners. + - **Bands and wealth.** The first member's pay puts the household in one of + three Table 3 income bands. Each household has wealth of its own, so in + stage 42's walk only a household's own clones and copies tie. + """ + + tables = base_id_tables(_HOUSEHOLDS) + household = tables.household[["household_id"]].copy() + sernum = household["household_id"].to_numpy() + household["region"] = np.asarray(_REGIONS)[sernum % 2] + for column in UK_CGT_INVESTABLE_WEALTH_COLUMNS: + household[column] = 0.0 + household[UK_CGT_INVESTABLE_WEALTH_COLUMNS[0]] = ( + 1_000.0 * ((37 * sernum) % 101) + sernum + ) + person = tables.person.copy() + member = person["person_id"].to_numpy() % 1_000 + owner = person["person_household_id"].to_numpy() + person["age"] = np.where( + member == 1, + np.where(owner % 3 == 1, 70, 45 + owner % 10), + np.where(member == 2, 44, 9), + ).astype("int64") + person["gender"] = np.where((owner + member) % 2 == 0, "MALE", "FEMALE") + person[SPI_RECIPIENT_ROLE_COLUMN] = member <= 2 + for column in UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS: + person[column] = 0.0 + pay = [BAND_INCOMES[_CGT_BANDS[(sernum // 6) % 3]] for sernum in owner.tolist()] + person["employment_income"] = np.where(member == 1, pay, 0.0) + return uk_national_frame( + person=person, + benunit=tables.benunit, + household=household, + household_weights=np.random.default_rng(_WEIGHT_SEED).uniform( + 200.0, 900.0, _HOUSEHOLDS + ), + time_period=_PERIOD, + weight_kind=WeightKind.DESIGN, + ) + + +def _tape() -> pd.DataFrame: + """An invented SPI tape: 480 records over six total-income levels, both + sexes and both of the frame's regions, all in age band 45 to 54 + (AGERANGE 4), with pay their main income. So every reserved band has tape + mass in every cell the frame's candidates fill.""" + + rng = np.random.default_rng(0) + records = [] + for index in range(480): + record = dict.fromkeys(SPI_DONOR_REQUIRED_COLUMNS, 0.0) + pay = _TAPE_PAY[index % 6] + record.update( + { + "SEX": float(1 + (index // 6) % 2), + "FACT": float(rng.integers(40, 80)), + # GORCODE 7 is London, 8 the South East. + "GORCODE": float(7 if (index // 12) % 2 == 0 else 8), + "AGERANGE": 4.0, + "PAY": pay, + "DIVIDENDS": 0.05 * pay, + "INCBBS": 0.01 * pay, + "MAINSRCE": 1.0, + } + ) + record["TEI"] = record["PAY"] + record["TII"] = record["DIVIDENDS"] + record["INCBBS"] + record["TI"] = record["TEI"] + record["TII"] + records.append(record) + return pd.DataFrame(records) + + +def _age_model() -> SPIDonorAgeModel: + """A flat single-year population, so the engine-free lane needs no engine.""" + + return SPIDonorAgeModel( + populations={ + (sex, age): 100.0 for sex in ("MALE", "FEMALE") for age in range(91) + }, + state_pension_age=_STATE_PENSION_AGE, + source="fixture", + ) + + +def _g1_stage() -> SourceStageSpec: + """A stand-in stage for G1: no stage declares it until the spine wires it.""" + + return SourceStageSpec( + stage=G1, + survey="Invented test stage", + source="Invented test stage", + grain="household", + artifacts=(), + operations=(), + outputs=UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, + ) + + +def _packaged_stage(name: str) -> SourceStageSpec: + spec = load_country_spec("uk") + assert spec.sources is not None + return spec.sources.stage_map()[name] + + +def _cgt_transform(top_cells) -> UKCGTSupportSplitStageTransform: + return UKCGTSupportSplitStageTransform( + stage=_packaged_stage(CGT), + distribution=support_distribution(top_cells), + parameters=PARAMETERS, + ) + + +# --- the runs --------------------------------------------------------------------- + +#: Each stage's exact-total correction. The runs record the weights each is +#: handed, so the rows it moved are where the stage's output differs from them. +_CORRECTIONS = { + SPI: (spi_support, "_importance_weights_with_exact_total"), + DONORS: (spi_band_donors, "_exact_incumbent_total"), + CGT: (cgt_support, "_exact_support_weights"), +} + + +@dataclass(frozen=True) +class _Run: + """One clone count's frames, each stage's result, and the rows each + stage's exact-total correction moved, by household id.""" + + clones: int + frames: Mapping[str, Frame] + results: Mapping[str, object] + moved: Mapping[str, frozenset[int]] + uncorrected: Mapping[str, pd.Series] + + +def _recording(function: Callable, calls: list[np.ndarray]) -> Callable: + def recorded(values, *args, **kwargs): + calls.append(np.array(values, dtype=float, copy=True)) + return function(values, *args, **kwargs) + + return recorded + + +def _run(base: Frame, clones: int, handed: Mapping[str, list[np.ndarray]]) -> _Run: + transforms = { + G1: UKGeographicExpandStageTransform(_g1_stage(), clones=clones), + SPI: UKSPISupportChannelStageTransform( + stage=_packaged_stage(SPI), + sample_fraction=_SAMPLE_FRACTION, + state_pension_age=_STATE_PENSION_AGE, + ), + DONORS: UKSPIIncomeBandDonorStageTransform( + "invented.tab", + stage=_packaged_stage(DONORS), + donor_table=_tape(), + band_taxpayers=_BAND_TAXPAYERS, + age_model=_age_model(), + ), + CGT: _cgt_transform(_CGT_TOP_CELLS), + } + frames: dict[str, Frame] = {} + results: dict[str, object] = {} + moved: dict[str, frozenset[int]] = {} + uncorrected: dict[str, pd.Series] = {} + frame = base + for stage, transform in transforms.items(): + for calls in handed.values(): + calls.clear() + frame = transform(frame) + frames[stage], results[stage] = frame, transform.last_result + if stage not in handed: + continue + calls = handed[stage] + assert len(calls) == 1, f"{stage} corrected its total {len(calls)} times." + ids = frame.table("household")["household_id"].to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float) + moved[stage] = frozenset(ids[weights != calls[0]].tolist()) + uncorrected[stage] = pd.Series(calls[0], index=ids) + return _Run(clones, frames, results, moved, uncorrected) + + +@pytest.fixture(scope="module") +def runs() -> dict[int, _Run]: + """The invented frame through G1 and stages 16, 17 and 42 at K = 1 and at + every K of :data:`_CLONE_COUNTS`, each exact-total correction recorded.""" + + handed: dict[str, list[np.ndarray]] = {stage: [] for stage in _CORRECTIONS} + with pytest.MonkeyPatch.context() as patch: + for stage, (module, name) in _CORRECTIONS.items(): + patch.setattr( + module, name, _recording(getattr(module, name), handed[stage]) + ) + base = _base_frame() + return {clones: _run(base, clones, handed) for clones in (1, *_CLONE_COUNTS)} + + +# --- helpers ------------------------------------------------------------------------ + + +def _weights(frame: Frame) -> pd.Series: + return pd.Series( + np.asarray(frame.weights_for("household").values, dtype=float), + index=frame.table("household")["household_id"].to_numpy(), + ) + + +def _clone_zero_ids(frame: Frame) -> set[int]: + household = frame.table("household") + index = household[ladder_clone_index_column("household")] + return set(household.loc[index.eq(0), "household_id"].tolist()) + + +def _mass_scale_report(runs: Mapping[int, _Run]) -> str: + """Stage 17's mass scale at each K, as booleans: equal to K = 1's, and one.""" + + one = runs[1].results[DONORS].mass_scale + return "stage 17's mass scale: " + "; ".join( + f"K={clones} equal to K=1's {run.results[DONORS].mass_scale == one}, " + f"one {run.results[DONORS].mass_scale == 1.0}" + for clones, run in runs.items() + ) + + +def _funding_strata(frame: Frame) -> dict[int, str]: + """Each household's stage-17 funding stratum (its region within its + clone), as the stage labels it.""" + + household = frame.table("household") + labels = spi_band_donors._funding_strata(household) + return dict(zip(household["household_id"].tolist(), labels.tolist(), strict=True)) + + +def _reached(one: _Run, many: _Run, stage: str) -> frozenset[int]: + """The one-clone households an exact-total correction moved by + ``stage``, at this stage or an earlier one, at one clone or in clone 0 + at K.""" + + done = _STAGES[1 : _STAGES.index(stage) + 1] + return frozenset( + household_id + for run in (one, many) + for earlier in done + for household_id in run.moved[earlier] + ) + + +def _in_a_reached_stratum(one: _Run, many: _Run, stage: str) -> frozenset[int]: + """From stage 17 on, the one-clone incumbents of a funding stratum whose + stage-16 mass holds a row stage 16's correction moved, at one clone or at + K, that no correction moved themselves (:func:`_reached`). + + The stratum's funding factor reads that mass, so their weights need not + be bitwise at K = 2; the factor rounds once at K and once at one clone, + within the stage's step budget. A donor's weight reads no stratum while + the mass scale is one. + """ + + if stage != DONORS: + return frozenset() + hit = set() + for run in (one, many): + strata = _funding_strata(run.frames[SPI]) + hit |= {strata[household_id] for household_id in run.moved[SPI]} + household = one.frames[DONORS].table("household") + labels = _funding_strata(one.frames[DONORS]) + incumbents = household.loc[ + ~household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR], "household_id" + ] + reached = _reached(one, many, stage) + return frozenset( + household_id + for household_id in incumbents.tolist() + if labels[household_id] in hit and household_id not in reached + ) + + +def _split_only(one: _Run, stage: str) -> frozenset[int]: + """The one-clone households whose weight at K is K = 1's over K through + the 1/K split's one rounding, every later step exact. + + - At G1, every household. + - At stage 16, the base households without a member at or over State + Pension age, whose share of 0.5 halves exactly. + - At stage 17, the donors, whose planned weight over K is scaled by a mass + scale of one. + """ + + frame = one.frames[stage] + household = frame.table("household") + ids = household["household_id"] + if stage == G1: + return frozenset(ids.tolist()) + if stage == SPI: + person = frame.table("person") + pension_age = person.loc[ + person["age"] >= _STATE_PENSION_AGE, "person_household_id" + ] + base = household[support_channel_column("household")].eq( + BASE_FRS_SUPPORT_CHANNEL + ) + return frozenset(ids[base & ~ids.isin(set(pension_age))].tolist()) + return frozenset(ids[household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR]].tolist()) + + +def _clone_index_problems(frame: Frame, clones: int) -> list[str]: + """Where ``frame`` breaks the clone-index invariant: a table without its + index, an index of another dtype or other values than exactly 0 to K - 1, + and members whose index is not their household's and benefit unit's.""" + + problems = [] + for entity in UK_NATIONAL_SCHEMA.entities: + column = ladder_clone_index_column(entity) + table = frame.table(entity) + if column not in table.columns: + problems.append(f"the {entity} table lacks {column!r}") + continue + if table[column].dtype != np.dtype("int64"): + problems.append(f"{column!r} is {table[column].dtype}, not int64") + if sorted(pd.unique(table[column]).tolist()) != list(range(clones)): + problems.append(f"{column!r} does not take exactly the values 0 to K - 1") + person = frame.table("person") + own = person[ladder_clone_index_column("person")].to_numpy() + for group in UK_NATIONAL_SCHEMA.group_entities: + units = frame.table(group).set_index(UK_NATIONAL_SCHEMA.id_column(group)) + theirs = person[UK_NATIONAL_SCHEMA.membership_column(group)].map( + units[ladder_clone_index_column(group)] + ) + off = int((theirs.to_numpy() != own).sum()) + if off: + problems.append( + f"{off} person row(s) carry another index than their {group}" + ) + return problems + + +def _copies_off_their_parents_clone(before: Frame, after: Frame, stage: str) -> int: + """How many households new at ``stage`` carry another clone index than + the household they copy. The parent is found by the copy's lineage + columns, never by id arithmetic: stages 16 and 17 write its id in the + copy's ``household_source_id``. At stage 42 the parent is the household + whose support lineage the copy restates.""" + + index = ladder_clone_index_column("household") + parents = before.table("household") + household = after.table("household") + new = household.loc[~household["household_id"].isin(set(parents["household_id"]))] + assert len(new), f"{stage} made no copy to check." + if stage == CGT: + keys = [ + support_source_id_column("household"), + support_clone_index_column("household"), + ] + by_key = parents.set_index(keys)[index] + assert by_key.index.is_unique, "The support lineage does not name one parent." + found = by_key.reindex(pd.MultiIndex.from_frame(new[keys])).to_numpy() + else: + by_id = parents.set_index("household_id")[index] + found = new[support_source_id_column("household")].map(by_id).to_numpy() + return int((pd.isna(found) | (found != new[index].to_numpy())).sum()) + + +def _copied(frame: Frame, clone: int) -> set[int]: + """The households stage 16 copied in ``clone``, by the copies' source ids.""" + + household = frame.table("household") + rows = household[support_channel_column("household")].eq( + SPI_SYNTHETIC_SUPPORT_CHANNEL + ) & household[ladder_clone_index_column("household")].eq(clone) + return set(household.loc[rows, support_source_id_column("household")].tolist()) + + +def _seating(frame: Frame, clone: int) -> dict[int, float]: + """Stage 17's seats in ``clone``: each donor's source household, and its band.""" + + household = frame.table("household") + donors = household.loc[ + household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR] + & household[ladder_clone_index_column("household")].eq(clone) + ] + return dict( + zip( + donors[support_source_id_column("household")].tolist(), + donors[SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN].tolist(), + strict=True, + ) + ) + + +def _carriers(frame: Frame, clone: int) -> set[int]: + """Stage 17's carriers in ``clone``, by the person each copy was made from.""" + + person = frame.table("person") + carriers = person.loc[ + person[PERSON_IS_SPI_INCOME_BAND_CARRIER] + & person[ladder_clone_index_column("person")].eq(clone) + ] + return set(carriers[support_source_id_column("person")].tolist()) + + +def _families(frame: Frame) -> pd.DataFrame: + """Stage 42's households, each with the root of its family: itself for a + root or an undivided household, else the household it copies, found by + its support lineage.""" + + household = frame.table("household") + keys = [ + support_source_id_column("household"), + support_clone_index_column("household"), + ] + roots = household.loc[household[CGT_SUPPORT_COPY_INDEX_COLUMN].eq(0)] + root_of = roots.set_index(keys)["household_id"] + assert root_of.index.is_unique, "The support lineage does not name one root." + root = root_of.reindex(pd.MultiIndex.from_frame(household[keys])).to_numpy() + assert not pd.isna(root).any(), "A copy's support lineage names no root." + return pd.DataFrame( + { + "household_id": household["household_id"].to_numpy(), + "root": root, + "source": household[SOURCE_HOUSEHOLD_ID_COLUMN].to_numpy(), + "support": household[support_clone_index_column("household")].to_numpy(), + "clone": household[ladder_clone_index_column("household")].to_numpy(), + "copies": household[CGT_SUPPORT_COPIES_COLUMN].to_numpy(), + "weight": np.asarray(frame.weights_for("household").values, dtype=float), + } + ) + + +def _band_row(result, band: int) -> dict[str, object]: + (row,) = [ + row for row in result.evidence()["bands"] if row["income_lower_bound"] == band + ] + return row + + +# --- reported first --------------------------------------------------------------- + + +def test_stage_17_mass_scale_is_reported_first_and_is_one_at_every_k(runs) -> None: + """Reported first: stage 17's mass scale at K = 1 and at each K. + + Stage 17 funds every clone's donors at one scale, the smallest any clone's + region allows. So clone 0's weights and funding factors are K = 1's over K + only while that scale is the one a one-clone spine takes: one at both, or + bound by a region of clone 0. On this frame every clone's regions fund + their donors in full, so the scale is exactly one at K = 1 and at every K. + The weight classes below rest on that. The seats are counted on the + eligible households in distinct sources, so the seating scale is exactly + K = 1's. + """ + + report = _mass_scale_report(runs) + one = runs[1].results[DONORS] + assert one.mass_scale == 1.0, report + for clones in _CLONE_COUNTS: + many = runs[clones].results[DONORS] + assert many.clone_count == clones + assert many.mass_scale == one.mass_scale, report + assert many.seating_scale == one.seating_scale, report + assert many.evidence()["seating_scale"] == one.evidence()["seating_scale"] + + +# --- exact ------------------------------------------------------------------------ + + +@pytest.mark.parametrize("clones", (1, *_CLONE_COUNTS)) +def test_g1_is_the_clone_first_stand_in(runs, clones: int) -> None: + """Bit for bit on every column the stand-in carries (the five id columns, + the household clone index and source id, and the frame's own columns, text + and booleans among them) and on the weights.""" + + base = runs[1].frames[G1] + expanded = runs[clones].frames[G1] + stand_in = clone_first_frame(_base_frame(), clones) + for entity in stand_in.entities: + table = stand_in.table(entity) + pd.testing.assert_frame_equal( + expanded.table(entity)[list(table.columns)], table, check_exact=True + ) + got, want = expanded.weights_for("household"), stand_in.weights_for("household") + assert got.kind is want.kind is WeightKind.DESIGN + assert got.values.tobytes() == want.values.tobytes() + assert len(expanded.table("household")) == clones * len(base.table("household")) + + +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_clone_zero_is_the_one_clone_frame_with_its_ids_and_g1_columns( + runs, clones: int +) -> None: + one, many = runs[1].frames[G1], runs[clones].frames[G1] + written = set(UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS) + for entity in UK_NATIONAL_SCHEMA.entities: + table = many.table(entity) + clone_zero = table.loc[table[ladder_clone_index_column(entity)].eq(0)] + # Clone-major: clone 0 is the frame's first rows, every column and + # dtype as at one clone, the ids and the source columns among them. + assert clone_zero.index.tolist() == list(range(len(one.table(entity)))) + pd.testing.assert_frame_equal(clone_zero, one.table(entity), check_exact=True) + written -= set(table.columns) + assert not written, "Some of G1's columns were not compared." + + +@pytest.mark.parametrize("clones", (1, *_CLONE_COUNTS)) +def test_every_frame_carries_k_clone_indices_and_every_copy_its_parents( + runs, clones: int +) -> None: + run = runs[clones] + for stage in _STAGES: + problems = _clone_index_problems(run.frames[stage], clones) + assert not problems, f"After {stage} at K={clones}: {problems}." + for before, stage in pairwise(_STAGES): + off = _copies_off_their_parents_clone( + run.frames[before], run.frames[stage], stage + ) + assert off == 0, ( + f"{off} household(s) {stage} made at K={clones} carry another clone " + "index than the household they copy." + ) + + +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_stage_16_copies_the_one_clone_households_in_clone_zero( + runs, clones: int +) -> None: + """The share divides by the distinct sources, so its probability is K = 1's, + and each household is drawn on a uniform keyed on its own id, which clone 0 + shares with K = 1. So clone 0 copies exactly K = 1's households, the copies' + ids included, and the other clones draw on their own ids.""" + + one, many = runs[1], runs[clones] + reference = _copied(one.frames[SPI], 0) + assert reference + assert _copied(many.frames[SPI], 0) == reference + copies_once = { + household_id + for household_id in _clone_zero_ids(one.frames[SPI]) + if household_id not in _clone_zero_ids(one.frames[G1]) + } + assert { + household_id + for household_id in _clone_zero_ids(many.frames[SPI]) + if household_id not in _clone_zero_ids(many.frames[G1]) + } == copies_once + for field in ( + "expected_spi_households", + "spi_source_households", + "spi_inclusion_probability", + ): + assert getattr(many.results[SPI], field) == getattr(one.results[SPI], field) + assert any( + _copied(many.frames[SPI], clone) != reference for clone in range(1, clones) + ) + + +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_stage_17_seats_the_one_clone_households_in_clone_zero( + runs, clones: int +) -> None: + """Clone 0 walks exactly K = 1's candidates on the same keyed uniforms with + the same seats, so it seats K = 1's households in K = 1's bands, with K = + 1's carriers. Every clone seats the same number in each band.""" + + one, many = runs[1].frames[DONORS], runs[clones].frames[DONORS] + report = _mass_scale_report(runs) + seating = _seating(one, 0) + assert seating, report + assert _seating(many, 0) == seating, report + assert _carriers(many, 0) == _carriers(one, 0), report + seats = Counter(seating.values()) + for clone in range(clones): + assert Counter(_seating(many, clone).values()) == seats, report + rows_once = {row["lower_bound"]: row for row in runs[1].results[DONORS].band_rows} + for row in runs[clones].results[DONORS].band_rows: + once = rows_once[row["lower_bound"]] + assert row["donor_households"] == clones * once["donor_households"] + assert row["carriers"] == clones * once["carriers"] + + +# --- tolerances --------------------------------------------------------------------- + + +def test_the_corrections_reach_rows_the_weight_classes_compare(runs) -> None: + """The premise that keeps the corrected rows' class from passing empty: on + this frame stage 16's and stage 17's exact-total corrections each move a + row of the one-clone frame, which every K's clone 0 is held to. A change + to either stage's arithmetic can move the corrections elsewhere; then pick + another :data:`_WEIGHT_SEED` at which this holds.""" + + one = runs[1] + for stage in (SPI, DONORS): + assert one.moved[stage] & _clone_zero_ids(one.frames[stage]), stage + # And stage 17 holds incumbents of a stratum stage 16's correction + # reached that no correction moved, so their class is exercised too. + for clones in _CLONE_COUNTS: + assert _in_a_reached_stratum(one, runs[clones], DONORS), clones + + +@pytest.mark.parametrize("stage", _WEIGHED_STAGES) +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_k_times_clone_zero_weights_nest_on_the_one_clone_weights( + runs, clones: int, stage: str +) -> None: + """K x each clone-0 weight against K = 1's, held exactly (as fractions): + + - **Rows an exact-total correction moved** (:func:`_reached`): within + ``rows x K x 2.2e-16`` relative. ``rows`` is the one-clone frame's + household count at the stage. On a row of average weight the bound is + about K ulps of the total: a correction moves its row by about an ulp of + the total, and K x a clone-0 weight scales that by K. + - **Stage 17's other incumbents of a stratum a correction reached** + (:func:`_in_a_reached_stratum`): within the stage's :data:`_STEP_ULPS` + ulps at every K, K = 2 included. + - **Every other row at K = 2**: bitwise, since halving is exact. + - **Every other row at K = 3, 7 and 15**: within one ulp where the 1/K split + alone rounds (:func:`_split_only`), and within :data:`_STEP_ULPS` ulps + elsewhere. + """ + + one, many = runs[1], runs[clones] + reference = _weights(one.frames[stage]) + clone_zero = _clone_zero_ids(many.frames[stage]) + assert clone_zero == set(reference.index), ( + f"Clone 0 at K={clones} after {stage} is not the one-clone frame's households." + ) + scaled = _weights(many.frames[stage]).loc[reference.index] + rows = len(reference) + reached = _reached(one, many, stage) + in_a_reached_stratum = _in_a_reached_stratum(one, many, stage) + split_only = _split_only(one, stage) + power_of_two = clones & (clones - 1) == 0 + corrected_bound = Fraction(rows * clones) * Fraction(_EPSILON) + classes: Counter[str] = Counter() + failures: Counter[str] = Counter() + for household_id, weight, expected in zip( + reference.index.tolist(), scaled.tolist(), reference.tolist(), strict=True + ): + difference = abs(Fraction(weight) * clones - Fraction(expected)) + if household_id in reached: + classes["corrected"] += 1 + if difference > corrected_bound * Fraction(expected): + failures["corrected rows beyond rows x K x 2.2e-16 relative"] += 1 + elif household_id in in_a_reached_stratum: + limit = _STEP_ULPS[stage] + classes["in a corrected stratum"] += 1 + if difference > limit * Fraction(float(np.spacing(expected))): + failures[f"rows in a corrected stratum beyond {limit} ulp(s)"] += 1 + elif power_of_two: + classes["bitwise"] += 1 + if difference: + failures["uncorrected rows not bitwise at a power-of-two K"] += 1 + else: + limit = 1 if household_id in split_only else _STEP_ULPS[stage] + classes[f"within {limit} ulp(s)"] += 1 + if difference > limit * Fraction(float(np.spacing(expected))): + failures[f"uncorrected rows beyond {limit} ulp(s)"] += 1 + assert not failures, ( + f"After {stage} at K={clones}, of {rows} clone-0 rows " + f"({dict(classes)}): {dict(failures)}. {_mass_scale_report(runs)}." + ) + uncorrected = sum( + count + for name, count in classes.items() + if name not in ("corrected", "in a corrected stratum") + ) + assert uncorrected, f"No uncorrected row was compared after {stage}." + + +# --- stage 42, approximately ------------------------------------------------------ + + +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_stage_42_nests_only_approximately(runs, clones: int) -> None: + """Stage 42 nests only approximately; these are the checks that hold. + + - **Roots.** Every row stage 17 wrote stays a root, so clone 0's roots are + K = 1's. + - **Family mass.** Each household's family (root and copies) keeps its + mass. So K x a clone-0 family's mass is K = 1's within the corrected + rows' class, wherever the walk stopped and however the family was + divided. + - **The divided band.** Band 0's whole pool is divided at every K, so the + selection nests there. Each clone-0 household in it is divided as at + K = 1, except where the K-ulp snap applies: a one-clone quotient within + K ulps (and the weights' tolerance) of an integer can count one copy + fewer. + - **The crossed band.** Band 37,700's walk stops past its support mass by + less than the row that crosses it, at K as at one clone. So the selected + masses differ by less than the heavier crossing row, not by the same + households. The walk can stop inside a household's clones, and since + ties are keyed per row, clone 0 need not be among the clones taken. The + messages say whether either happened. + - **Rule B.** The exact-total correction moves at most one row, within the + corrected rows' class of its uncorrected weight; it can be a copy (the + last test). Every other root and copy of a divided household weighs + bitwise w / n. + """ + + one, many = runs[1], runs[clones] + rows = len(one.frames[CGT].table("household")) + tolerance = rows * clones * _EPSILON + families = {k: _families(run.frames[CGT]) for k, run in ((1, one), (clones, many))} + + # Every selected household here is divided (K x its weight exceeds 60), + # so a household is selected exactly where it has more than one copy. + for k, run in ((1, one), (clones, many)): + roots = families[k].loc[families[k]["household_id"] == families[k]["root"]] + assert run.results[CGT].households_selected == int((roots["copies"] > 1).sum()) + + # Roots and family mass. + masses = { + k: { + root: math.fsum(group.tolist()) + for root, group in family.groupby("root")["weight"] + } + for k, family in families.items() + } + many_roots = families[clones].loc[ + (families[clones]["household_id"] == families[clones]["root"]) + & families[clones]["clone"].eq(0), + "household_id", + ] + assert set(many_roots.tolist()) == set(masses[1]) + off = sum( + 1 + for root, mass in masses[1].items() + if abs(clones * masses[clones][root] - mass) > tolerance * mass + ) + assert off == 0, f"{off} clone-0 family mass(es) at K={clones} do not nest." + + # The divided band: whole pool at every K, copy counts as at one clone. + weights_17 = { + k: _weights(run.frames[DONORS]) for k, run in ((1, one), (clones, many)) + } + roots_once = families[1].loc[families[1]["household_id"] == families[1]["root"]] + bands = cgt_support_income_band( + one.frames[DONORS].table("person"), + household_ids=roots_once["household_id"].to_numpy(), + parameters=PARAMETERS, + ) + for result in (one.results[CGT], many.results[CGT]): + row = _band_row(result, _EXHAUSTED_BAND) + assert row["pool_exhausted"] + assert row["households_selected"] == row["pool_households"] > 0 + copies = { + k: family.set_index("household_id")["copies"] for k, family in families.items() + } + snapped = miscounted = 0 + for household_id in roots_once.loc[bands == _EXHAUSTED_BAND, "household_id"]: + quotient = weights_17[1][household_id] / CGT_SUPPORT_MAXIMUM_COPY_WEIGHT + nearest = round(quotient) + window = clones * float(np.spacing(float(nearest))) + tolerance * quotient + counted = int(copies[clones][household_id]) - int(copies[1][household_id]) + if abs(quotient - nearest) <= window: + snapped += 1 + miscounted += abs(counted) > 1 + else: + miscounted += counted != 0 + assert miscounted == 0, ( + f"{miscounted} clone-0 household(s) of the divided band at K={clones} " + f"are divided otherwise than at K=1 ({snapped} in the snap window)." + ) + + # The crossed band: the same mass within the heavier crossing row. Its + # households at K are grouped by what their clones share, the survey + # household and the support copy, to say where the walk stopped. + crossed = { + k: _band_row(run.results[CGT], _CROSSING_BAND) + for k, run in ((1, one), (clones, many)) + } + roots = families[clones].loc[ + families[clones]["household_id"] == families[clones]["root"] + ] + in_band = cgt_support_income_band( + many.frames[DONORS].table("person"), + household_ids=roots["household_id"].to_numpy(), + parameters=PARAMETERS, + ) + taken = roots.loc[in_band == _CROSSING_BAND].assign( + taken=lambda table: table["copies"] > 1 + ) + by_household = taken.groupby(["source", "support"])["taken"] + partial = by_household.any() & ~by_household.all() + clone_zero = taken.loc[taken["clone"].eq(0)].set_index(["source", "support"]) + left = partial & clone_zero["taken"].reindex(partial.index).eq(False) + walk = ( + f"at K={clones} the walk stopped inside {int(partial.sum())} household(s)' " + f"clones, {int(left.sum())} of them leaving clone 0" + ) + for row in crossed.values(): + assert not row["pool_exhausted"], walk + assert row["selected_mass"] >= row["support_mass"], walk + assert abs(crossed[clones]["selected_mass"] - crossed[1]["selected_mass"]) < max( + crossed[1]["heaviest_selected_weight"], + crossed[clones]["heaviest_selected_weight"], + ), walk + + # Rule B: at most one row moved, within the corrected class; every other + # root and copy of a divided household at bitwise w / n. + for k, run in ((1, one), (clones, many)): + moved = run.moved[CGT] + assert len(moved) <= 1, f"{len(moved)} rows took the correction at K={k}." + weights = _weights(run.frames[CGT]) + for household_id in moved: + before = run.uncorrected[CGT][household_id] + assert abs(weights[household_id] - before) <= ( + rows * k * _EPSILON * before + ), f"Rule B's row at K={k} moved beyond the corrected rows' class." + family = families[k] + divided = family.loc[family["copies"] > 1] + expected = ( + weights_17[k].reindex(divided["root"]).to_numpy() + / divided["copies"].to_numpy() + ) + off_rule = set( + divided.loc[divided["weight"].to_numpy() != expected, "household_id"] + ) + assert off_rule <= moved, ( + f"{len(off_rule - moved)} row(s) of divided households at K={k} are off " + "w / n without taking the correction." + ) + + +# --- rule B on a copy, through G1 --------------------------------------------------- +# +# test_uk_cgt_support.py's nesting scenario (test_support's NESTING_* data): +# nine one-person households. Band 0 holds exact multiples of 60, a weight one +# ulp below 300, a light one and a non-multiple, and asks for far more than its +# pool, so the whole pool is divided at every clone count. Band 37,700 asks for +# nothing, so its household (9,000) and the light one are the households the +# split leaves whole. At 7 and 13 clones none of their clones can carry the +# correction, so spi_support's shared fallback puts it on the last row: the +# last copy of the 7,620 household's last clone (127 rows, copy index 126). + + +@pytest.mark.parametrize("clones", (1, 2, 3, 7, 11, 13, 15)) +def test_rule_b_puts_the_correction_on_a_copy_at_seven_and_thirteen_clones( + monkeypatch, clones: int +) -> None: + base = support_frame( + weights=NESTING_WEIGHTS, wealth=NESTING_WEALTH, incomes=NESTING_INCOMES + ) + handed: list[np.ndarray] = [] + monkeypatch.setattr( + cgt_support, + "_exact_support_weights", + _recording(cgt_support._exact_support_weights, handed), + ) + expanded = expand_uk_geographic_clones(base, clones=clones).frame + split = _cgt_transform(NESTING_TOP_CELLS)(expanded) + stand_in = _cgt_transform(NESTING_TOP_CELLS)(clone_first_frame(base, clones)) + assert len(handed) == 2 + values = handed[0] + weights = np.asarray(split.weights_for("household").values, dtype=float) + + # G1 is a drop-in for the stand-in here: the split weighs its output's rows + # bit for bit as it weighs the stand-in's, and keeps the total exactly. + assert ( + weights.tobytes() + == np.asarray(stand_in.weights_for("household").values).tobytes() + ) + assert ( + split.weights_for("household").total == expanded.weights_for("household").total + ) + moved = np.flatnonzero(weights != values).tolist() + if clones not in (7, 13): + assert moved == [] + return + assert moved == [len(weights) - 1] + last = split.table("household").iloc[-1] + assert bool(last[HOUSEHOLD_IS_CGT_SUPPORT_COPY]) + assert int(last[SOURCE_HOUSEHOLD_ID_COLUMN]) == 4 + assert int(last[ladder_clone_index_column("household")]) == clones - 1 + assert int(last[CGT_SUPPORT_COPY_INDEX_COLUMN]) == 126 + # About an ulp of the total off w / n: below 1e-12 of the copy's weight. + assert abs(weights[-1] - values[-1]) < 1e-12 * values[-1] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py index beffddd324..2dec44d4ff 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -673,7 +673,10 @@ def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: } #: Every module that mints, bounds or decodes spine copy ids; none may also #: name the retired rule. Each reads the copy-id table, save the band -#: donors, which mint through the support channel's clone helper. +#: donors, which mint through the support channel's clone helper. The +#: geographic expansion is the one reader among the clone-first steps: it +#: mints the clones and exports their limit, and the geography identity reads +#: clone paths from the lineage columns, never from ids. _COPY_ID_MODULES = frozenset( { *( @@ -683,6 +686,7 @@ def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: "cgt_structure", "cgt_support", "frs_spine", + "geographic_expand", "graph_kernels", "high_value_home_support", "spi_band_donors", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_expand.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_expand.py new file mode 100644 index 0000000000..0b338321c9 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_expand.py @@ -0,0 +1,552 @@ +"""The geographic expansion, G1 of the clone-first spine (microcosm#931). + +G1 clones every FRS household K times through the shared clone expansion core +under the UK's declaration, writing each household's source columns first. +These tests hold it to the clone-first stand-in (``clone_first_frame`` in +test_support/microcosm_build/uk_clone_first_frames.py, written without the +core) at K = 1, 2, 3, 7 and 15, and check the one-clone frame, the clone-major +ids, indices, sources and weights at K = 3, the dtypes, the mass record, the +evidence and every refusal. Invented frames only; every refusal prints counts, +names and declared constants, never an id or a weight. +""" + +from __future__ import annotations + +import re + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.clone_expansion import CloneDeclaration, CloneIdRule +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table, geographic_expand +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.geographic_clones import ( + geographic_clone_count, + ladder_clone_index_column, +) +from microcosm.build.uk_runtime.geographic_expand import ( + GEOGRAPHIC_CLONE_LIMIT, + GEOGRAPHIC_COPY_ID_FIELD, + GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON, + GEOGRAPHIC_EXPAND_STAGE_NAME, + SOURCE_HOUSEHOLD_KEY_COLUMN, + SOURCE_YEAR_COLUMN, + UK_GEOGRAPHIC_CLONES, + UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, + UKGeographicExpandStageTransform, + expand_uk_geographic_clones, + mint_geographic_clone_ids, + require_room_for_geographic_clones, +) +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_SUPPORT_BRANCH, + UK_COPY_LINEAGE, +) +from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.receipt_counts import rounded_count +from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights +from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN +from test_support.microcosm_build.uk_clone_first_frames import ( + base_id_tables, + clone_first_frame, + with_last_household_moved, +) + +_CLONE_COUNTS = (1, 2, 3, 7, 15) +#: The five id columns, by table. +_ID_COLUMNS = { + "person": ("person_id", "person_household_id", "person_benunit_id"), + "benunit": ("benunit_id",), + "household": ("household_id",), +} +#: The pinned FRS survey year, so the frames' time period is the one the +#: geography identity maps to a vintage. +_PERIOD = str(load_uk_frs_release().survey_year) + + +def _base_frame(households: int = 12, *, time_period: str = _PERIOD) -> Frame: + """Invented FRS-shaped ids (test_support's ``base_id_tables``) without any + lineage, a text and a float column, and invented design weights.""" + + tables = base_id_tables(households) + household = tables.household[["household_id"]].copy() + household["region"] = [ + "WALES" if sernum % 3 == 0 else "NORTH_EAST" + for sernum in household["household_id"].tolist() + ] + person = tables.person.copy() + person["employment_income"] = 875.5 * (person["person_id"] % 1000) + person[ + "person_household_id" + ].astype(float) + return uk_national_frame( + person=person, + benunit=tables.benunit, + household=household, + household_weights=np.random.default_rng(931).uniform(400.0, 3000.0, households), + time_period=time_period, + ) + + +def _stage(**overrides: object) -> SourceStageSpec: + """A stand-in stage for G1: no stage declares it until the spine wires it.""" + + fields: dict[str, object] = { + "stage": GEOGRAPHIC_EXPAND_STAGE_NAME, + "survey": "Invented test stage", + "source": "Invented test stage", + "grain": "household", + "artifacts": (), + "operations": (), + "outputs": UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, + } + fields.update(overrides) + return SourceStageSpec(**fields) + + +def _with_column(frame: Frame, entity: str, column: str) -> Frame: + tables = {name: frame.table(name).copy() for name in frame.entities} + tables[entity][column] = np.zeros(len(tables[entity]), dtype=np.int64) + return Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _with_earlier_mass_record(frame: Frame) -> Frame: + """``frame`` with one invented household mass record, as an earlier step + would leave, so a test sees what G1 does to the log it is handed.""" + + total = frame.weights_for("household").total + record = MassChangeRecord( + entity="household", + old_total=2.0 * total, + new_total=total, + declared_factor=0.5, + reason="An invented earlier step halved household mass.", + ) + return Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=(*frame.mass_log, record), + metadata=frame.metadata, + ) + + +def _weight_texts(frame: Frame) -> list[str]: + return [repr(float(w)) for w in frame.weights_for("household").values] + + +# --- what G1 writes -------------------------------------------------------------- + + +def test_one_clone_adds_no_row_and_changes_no_column() -> None: + frame = _with_earlier_mass_record(_base_frame()) + expanded = expand_uk_geographic_clones(frame, clones=1).frame + for entity in frame.entities: + before, after = frame.table(entity), expanded.table(entity) + assert len(after) == len(before) + pd.testing.assert_frame_equal( + after[list(before.columns)], before, check_exact=True + ) + index = after[ladder_clone_index_column(entity)] + assert index.dtype == np.int64 and not index.any() + household = expanded.table("household") + ids = household["household_id"] + assert household[SOURCE_HOUSEHOLD_ID_COLUMN].to_numpy().tobytes() == ( + ids.to_numpy().tobytes() + ) + assert household[SOURCE_YEAR_COLUMN].dtype == np.int64 + assert set(household[SOURCE_YEAR_COLUMN].tolist()) == {int(_PERIOD)} + assert household[SOURCE_HOUSEHOLD_KEY_COLUMN].tolist() == [ + f"{_PERIOD}:{household_id}" for household_id in ids.tolist() + ] + weights = expanded.weights_for("household") + assert weights.kind is WeightKind.DESIGN + assert weights.values.tobytes() == frame.weights_for("household").values.tobytes() + # The log it is handed, and one record more. + assert len(frame.mass_log) == 1 + assert len(expanded.mass_log) == len(frame.mass_log) + 1 + assert expanded.mass_log[:-1] == frame.mass_log + assert uk_time_period(expanded) == uk_time_period(frame) + assert dict(expanded.metadata) == dict(frame.metadata) + # Exactly the declared columns are added. + added = { + column + for entity in frame.entities + for column in expanded.table(entity).columns + if column not in frame.table(entity).columns + } + assert added == set(UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS) + + +def test_three_clones_are_clone_major_with_exactly_three_indices() -> None: + frame = _base_frame() + clones = 3 + expanded = expand_uk_geographic_clones(frame, clones=clones).frame + validate_uk_national_frame(expanded) + radix = copy_id_table.radix(GEOGRAPHIC_COPY_ID_FIELD) + for entity, columns in _ID_COLUMNS.items(): + table = expanded.table(entity) + rows = len(frame.table(entity)) + index = table[ladder_clone_index_column(entity)].to_numpy() + # Clone-major: clone k's rows are the frame's rows, in their order. + assert index.tobytes() == np.repeat(np.arange(clones), rows).tobytes() + assert sorted(set(index.tolist())) == list(range(clones)) + for column in columns: + original = frame.table(entity)[column].to_numpy() + assert table[column].dtype == np.int64 + assert table[column].tolist() == [ + int(value) + k * radix for k in range(clones) for value in original + ] + # Every member carries its household's and its benefit unit's clone. + person = expanded.table("person") + for group in UK_NATIONAL_SCHEMA.group_entities: + units = expanded.table(group).set_index(UK_NATIONAL_SCHEMA.id_column(group)) + assert np.array_equal( + units.loc[ + person[UK_NATIONAL_SCHEMA.membership_column(group)], + ladder_clone_index_column(group), + ].to_numpy(), + person[ladder_clone_index_column("person")].to_numpy(), + ) + household = expanded.table("household") + assert geographic_clone_count(household) == clones + # Each clone keeps its household's source columns. + base_ids = frame.table("household")["household_id"].tolist() + assert household[SOURCE_HOUSEHOLD_ID_COLUMN].tolist() == base_ids * clones + assert ( + household[SOURCE_HOUSEHOLD_KEY_COLUMN].tolist() + == [f"{_PERIOD}:{household_id}" for household_id in base_ids] * clones + ) + # One float per household, the same in each clone, summing to the frame's. + original = frame.weights_for("household").values + weights = expanded.weights_for("household").values.reshape(clones, -1) + for clone in weights: + assert clone.tobytes() == (original / clones).tobytes() + assert np.isclose(weights.sum(), original.sum(), rtol=1e-9, atol=0.0) + record = expanded.mass_log[-1] + assert (record.entity, record.declared_factor) == ("household", 1.0) + assert record.reason == GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON + + +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_g1_is_the_clone_first_stand_in(clones: int) -> None: + """Bit for bit on every column the stand-in carries (the five id columns, + the household clone index and source id, and the copied columns), in the + stand-in's dtypes, and on the weights.""" + + frame = _base_frame() + expanded = expand_uk_geographic_clones(frame, clones=clones).frame + stand_in = clone_first_frame(frame, clones) + for entity in stand_in.entities: + table = stand_in.table(entity) + pd.testing.assert_frame_equal( + expanded.table(entity)[list(table.columns)], table, check_exact=True + ) + for column in _ID_COLUMNS[entity]: + assert ( + expanded.table(entity)[column].to_numpy().tobytes() + == table[column].to_numpy().tobytes() + ) + got, want = expanded.weights_for("household"), stand_in.weights_for("household") + assert got.kind is want.kind is WeightKind.DESIGN + assert got.values.tobytes() == want.values.tobytes() + + +def test_the_source_key_is_python_strings_in_an_object_column() -> None: + """Whatever pandas infers for text: the dtype the SPI support channel's + own write gives with string inference off, as in the legacy plan, so + that plan's object-string audit keeps the column.""" + + for infer in (True, False): + with pd.option_context("future.infer_string", infer): + expanded = expand_uk_geographic_clones(_base_frame(), clones=3).frame + key = expanded.table("household")[SOURCE_HOUSEHOLD_KEY_COLUMN] + assert key.dtype == object + assert {type(value) for value in key.tolist()} == {str} + + +def test_the_mass_record_names_no_clone_count() -> None: + frame = _base_frame() + records = [ + expand_uk_geographic_clones(frame, clones=clones).frame.mass_log[-1] + for clones in _CLONE_COUNTS + ] + assert {record.reason for record in records} == { + GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON + } + assert not re.search(r"\d", GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON) + total = frame.weights_for("household").total + for record in records: + assert (record.entity, record.declared_factor) == ("household", 1.0) + assert np.isclose(record.old_total, total, rtol=1e-9, atol=0.0) + assert np.isclose(record.new_total, total, rtol=1e-9, atol=0.0) + + +def test_the_evidence_gives_k_the_field_rounded_sources_and_the_split() -> None: + frame = _base_frame(households=26) + evidence = { + clones: expand_uk_geographic_clones(frame, clones=clones).evidence() + for clones in (1, 3) + } + assert evidence[3] == { + "stage": GEOGRAPHIC_EXPAND_STAGE_NAME, + "clone_count": 3, + "copy_id_field": GEOGRAPHIC_COPY_ID_FIELD, + "copy_id_radix": copy_id_table.radix(GEOGRAPHIC_COPY_ID_FIELD), + # Distinct source households, rounded at the receipts' floor: the + # same at every K. + "source_households": rounded_count(26), + "weight_split": "equal", + "weight_kind": "design", + } + assert {**evidence[1], "clone_count": 3} == evidence[3] + small = expand_uk_geographic_clones(_base_frame(households=2), clones=3) + assert small.evidence()["source_households"] == "<3" + # The result's repr prints no count. + assert repr(small) == "UKGeographicExpandResult(clone_count=3)" + + +def test_the_transform_runs_the_step_and_records_its_evidence() -> None: + frame = _base_frame() + transform = UKGeographicExpandStageTransform(_stage(), clones=3) + with pytest.raises(RuntimeError, match="completed stage run"): + transform.checkpoint_metadata() + expanded = transform(frame) + direct = expand_uk_geographic_clones(frame, clones=3) + for entity in frame.entities: + pd.testing.assert_frame_equal( + expanded.table(entity), direct.frame.table(entity), check_exact=True + ) + assert transform.checkpoint_metadata() == {"evidence": direct.evidence()} + assert transform.output_columns() == UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS + + +def test_the_declaration_is_the_copy_id_tables_field_through_named_functions() -> None: + """The mint and the bound check are this module's named functions, so the + derived binding rule follows them into the copy-id table; the field is + the copy lineage's geographic branch.""" + + rule = UK_GEOGRAPHIC_CLONES.ids + assert rule.mint is mint_geographic_clone_ids + assert rule.require_below is require_room_for_geographic_clones + assert rule.bound == GEOGRAPHIC_CLONE_LIMIT + assert GEOGRAPHIC_CLONE_LIMIT == copy_id_table.ordinal_bound( + GEOGRAPHIC_COPY_ID_FIELD + ) + assert GEOGRAPHIC_COPY_ID_FIELD == GEOGRAPHIC_SUPPORT_BRANCH + assert dict(UK_GEOGRAPHIC_CLONES.clone_index_columns) == { + entity: ladder_clone_index_column(entity) + for entity in UK_NATIONAL_SCHEMA.entities + } + assert dict(UK_GEOGRAPHIC_CLONES.source_id_columns) == { + "household": SOURCE_HOUSEHOLD_ID_COLUMN + } + assert UK_GEOGRAPHIC_CLONES.weight_kinds == frozenset({WeightKind.DESIGN}) + assert UK_GEOGRAPHIC_CLONES.reason == GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON + ids = np.arange(1, 6, dtype=np.int64) + assert ( + mint_geographic_clone_ids(ids, 2).tolist() + == (ids + 2 * copy_id_table.radix(GEOGRAPHIC_COPY_ID_FIELD)).tolist() + ) + + +# --- refusals --------------------------------------------------------------------- + + +@pytest.mark.parametrize("clones", [0, GEOGRAPHIC_CLONE_LIMIT + 1, True, 2.0, "3"]) +def test_a_clone_count_outside_the_field_s_bound_is_refused(clones: object) -> None: + message = r"clone count from 1 to 100, the 'geographic_support' copy-id field" + with pytest.raises(ValueError, match=message): + UKGeographicExpandStageTransform(_stage(), clones=clones) + with pytest.raises(ValueError, match=message): + expand_uk_geographic_clones(_base_frame(), clones=clones) + + +def test_the_largest_clone_count_expands() -> None: + expanded = expand_uk_geographic_clones( + _base_frame(households=3), clones=GEOGRAPHIC_CLONE_LIMIT + ).frame + assert geographic_clone_count(expanded.table("household")) == 100 + + +@pytest.mark.parametrize( + ("entity", "column"), + [ + *( + (entity, ladder_clone_index_column(entity)) + for entity in UK_NATIONAL_SCHEMA.entities + ), + ("household", SOURCE_HOUSEHOLD_ID_COLUMN), + ("household", SOURCE_YEAR_COLUMN), + ("household", SOURCE_HOUSEHOLD_KEY_COLUMN), + *( + ("household", column) + for column in UK_COPY_LINEAGE.copy_columns + if column != ladder_clone_index_column("household") + ), + ], +) +def test_a_frame_with_a_clone_index_source_or_copy_column_is_refused( + entity: str, column: str +) -> None: + frame = _with_column(_base_frame(), entity, column) + with pytest.raises(ValueError, match="before any copy step") as error: + expand_uk_geographic_clones(frame, clones=2) + assert repr(column) in str(error.value) + + +def test_a_second_expansion_is_refused() -> None: + once = expand_uk_geographic_clones(_base_frame(), clones=2).frame + with pytest.raises(ValueError, match="already carries") as error: + expand_uk_geographic_clones(once, clones=2) + for column in UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS: + assert repr(column) in str(error.value) + + +def test_an_id_in_the_field_is_refused_by_count() -> None: + radix = copy_id_table.radix(GEOGRAPHIC_COPY_ID_FIELD) + moved = with_last_household_moved(_base_frame(), radix) + with pytest.raises( + ValueError, + match="1 of the household_id values reach the 'geographic_support' copy-id", + ) as error: + expand_uk_geographic_clones(moved, clones=2) + moved_id = int(moved.table("household")["household_id"].iloc[-1]) + assert str(moved_id) not in str(error.value) + + +def test_importance_weights_are_refused() -> None: + frame = _base_frame() + importance = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + { + "household": Weights( + frame.weights_for("household").values, WeightKind.IMPORTANCE + ) + }, + frame.strata, + metadata=frame.metadata, + ) + with pytest.raises(ValueError, match=r"accepts \['design'\] weights") as error: + expand_uk_geographic_clones(importance, clones=2) + for text in _weight_texts(frame): + assert text not in str(error.value) + + +def test_weights_on_another_entity_are_refused() -> None: + frame = _base_frame() + people = len(frame.table("person")) + weighted = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + { + "person": Weights(np.full(people, 2.5), WeightKind.DESIGN), + "household": frame.weights_for("household"), + }, + frame.strata, + metadata=frame.metadata, + ) + with pytest.raises( + ValueError, match="splits household weights alone, as the UK weights" + ) as error: + expand_uk_geographic_clones(weighted, clones=2) + assert "'person'" in str(error.value) + for text in _weight_texts(frame): + assert text not in str(error.value) + + +def _mint_every_clone_at_ordinal_one(ids: np.ndarray, k: int) -> np.ndarray: + return copy_id_table.mint(ids, GEOGRAPHIC_COPY_ID_FIELD, 1) + + +def test_a_mint_that_does_not_put_each_clone_above_the_last_is_refused( + monkeypatch, +) -> None: + patched = CloneDeclaration( + ids=CloneIdRule( + bound=GEOGRAPHIC_CLONE_LIMIT, + mint=_mint_every_clone_at_ordinal_one, + require_below=require_room_for_geographic_clones, + ), + clone_index_columns=dict(UK_GEOGRAPHIC_CLONES.clone_index_columns), + weight_kinds=UK_GEOGRAPHIC_CLONES.weight_kinds, + reason=UK_GEOGRAPHIC_CLONES.reason, + source_id_columns=dict(UK_GEOGRAPHIC_CLONES.source_id_columns), + ) + monkeypatch.setattr(geographic_expand, "UK_GEOGRAPHIC_CLONES", patched) + # Two clones mint one clone above the frame, which this mint does. + expand_uk_geographic_clones(_base_frame(), clones=2) + with pytest.raises(ValueError, match="above the clone before's"): + expand_uk_geographic_clones(_base_frame(), clones=3) + + +@pytest.mark.parametrize("clones", [1, 2]) +def test_ids_that_are_not_int64_are_refused_at_every_clone_count(clones: int) -> None: + """The copy-id table mints int64 ids, so the core refuses narrower ids + from two clones on; G1 refuses them at one clone too, so every clone + count takes the same frames.""" + + frame = _base_frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for entity, columns in _ID_COLUMNS.items(): + for column in columns: + tables[entity][column] = tables[entity][column].astype(np.int32) + narrow = Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + with pytest.raises( + ValueError, match="takes int64 id and membership columns, the dtype" + ) as error: + expand_uk_geographic_clones(narrow, clones=clones) + message = str(error.value) + for columns in _ID_COLUMNS.values(): + for column in columns: + assert f"{column!r} holds int32" in message + for person_id in frame.table("person")["person_id"].tolist(): + assert str(person_id) not in message + + +def test_a_time_period_that_is_not_a_year_is_refused() -> None: + with pytest.raises(ValueError, match="must name a year"): + expand_uk_geographic_clones(_base_frame(time_period="FY24"), clones=2) + + +def test_a_non_frame_is_refused() -> None: + with pytest.raises(TypeError, match="clones a Frame"): + expand_uk_geographic_clones(_base_frame().table("household"), clones=2) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"stage": "spi_support_channel"}, "Expected stage 'geographic_expand'"), + ({"grain": "person"}, "grain must be 'household'"), + ({"outputs": UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS[:-1]}, "must declare outputs"), + ({"rewrites": ("region",)}, "and no rewrites"), + ], +) +def test_a_stage_that_is_not_g1_s_is_refused( + overrides: dict[str, object], message: str +) -> None: + with pytest.raises(ValueError, match=message): + UKGeographicExpandStageTransform(_stage(**overrides), clones=2) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_identity.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_identity.py new file mode 100644 index 0000000000..330060c509 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_identity.py @@ -0,0 +1,409 @@ +"""The geography identity, G1b of the clone-first spine (microcosm#931). + +G1b keys every household of the geographic expansion's frame by its source, +FRS vintage, true source and copy path, as the dense line's +``uk.full.identity`` keys it. These tests hold clone 0's key, and clone k's, +to the dense identity kernel itself on the same households, and check the +clone path, uniqueness, nesting across K, the dtype, the vintage read from +frs_release.json, the evidence and every refusal. Invented frames only; no +refusal prints a key, an id or a year of a row. +""" + +from __future__ import annotations + +import inspect +from dataclasses import replace +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.atomic_area_support import IDENTITY_COLUMN +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.frs_release import load_uk_frs_release +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.geographic_expand import ( + SOURCE_HOUSEHOLD_KEY_COLUMN, + SOURCE_YEAR_COLUMN, + expand_uk_geographic_clones, +) +from microcosm.build.uk_runtime.geographic_identity import ( + GEOGRAPHIC_IDENTITY_INPUTS, + GEOGRAPHIC_IDENTITY_SOURCE, + GEOGRAPHIC_IDENTITY_STAGE_NAME, + UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS, + UKGeographicIdentityStageTransform, + frs_vintages_by_source_year, + key_uk_households_by_clone_path, +) +from microcosm.build.uk_runtime.graph_population import ( + UKGeographyIdentityKernel, + append_uk_population_nodes, +) +from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_SUPPORT_BRANCH +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.receipt_counts import rounded_count +from microcosm.frame import Frame +from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN +from microcosm.graph.population import dtype_for_token +from test_support.microcosm_build.uk_clone_first_frames import base_id_tables + +_RELEASE = load_uk_frs_release() +_PERIOD = str(_RELEASE.survey_year) +#: Years the release pins no vintage for: before its survey year and after +#: its base year. +_UNPINNED_EARLIER = min(_RELEASE.survey_year, _RELEASE.base_year) - 1 +_UNPINNED_LATER = max(_RELEASE.survey_year, _RELEASE.base_year) + 1 +_CLONE = ladder_clone_index_column("household") +#: The dense identity's lineage columns at their values on a base household: +#: no SPI support copy, CGT copy, incidence clone or residential arm. +_BASE_LINEAGE = { + "household_support_channel": "frs", + "household_support_clone_index": 0, + "household_is_cgt_support_copy": False, + "cgt_support_copy_index": 0, + "household_is_capital_gains_clone": False, + "household_is_cgt_residential_clone": False, + "cgt_residential_clone_index": 0, +} + + +def _base_frame(households: int = 9, *, time_period: str = _PERIOD) -> Frame: + """Invented FRS-shaped ids (test_support's ``base_id_tables``) without any + lineage, at invented design weights.""" + + tables = base_id_tables(households) + household = tables.household[["household_id"]].copy() + household["region"] = "WALES" + return uk_national_frame( + person=tables.person, + benunit=tables.benunit, + household=household, + household_weights=np.random.default_rng(1931).uniform( + 400.0, 3000.0, households + ), + time_period=time_period, + ) + + +def _expanded(clones: int, households: int = 9, **kwargs: str) -> Frame: + return expand_uk_geographic_clones( + _base_frame(households, **kwargs), clones=clones + ).frame + + +def _with_household(frame: Frame, household: pd.DataFrame) -> Frame: + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = household + return Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _stage(**overrides: object) -> SourceStageSpec: + """A stand-in stage for G1b: no stage declares it until the spine wires it.""" + + fields: dict[str, object] = { + "stage": GEOGRAPHIC_IDENTITY_STAGE_NAME, + "survey": "Invented test stage", + "source": "Invented test stage", + "grain": "household", + "artifacts": (), + "operations": (), + "outputs": UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS, + } + fields.update(overrides) + return SourceStageSpec(**fields) + + +def _dense_keys(household: pd.DataFrame) -> list[str]: + """The dense line's own identity kernel on these households, given the + lineage a base household carries there and the full build's params.""" + + dense = household.assign(**_BASE_LINEAGE) + context = SimpleNamespace( + tables={"household": dense}, + params={"source": "frs", "source_vintage": _RELEASE.vintage}, + ) + result = UKGeographyIdentityKernel().run(context) + (keys,) = result.columns.values() + return keys.tolist() + + +def _keys(frame: Frame) -> list[str]: + return frame.table("household")[IDENTITY_COLUMN].tolist() + + +# --- the keys ---------------------------------------------------------------------- + + +@pytest.mark.parametrize("clones", [1, 3]) +def test_every_clone_keys_as_the_dense_identity_keys_that_clone(clones: int) -> None: + expanded = _expanded(clones) + keyed = key_uk_households_by_clone_path(expanded).frame + assert _keys(keyed) == _dense_keys(expanded.table("household")) + + +def test_clone_zero_keys_on_its_source_and_clone_k_on_its_geographic_branch() -> None: + expanded = _expanded(3) + household = expanded.table("household") + keys = _keys(key_uk_households_by_clone_path(expanded).frame) + for key, source, clone in zip( + keys, + household[SOURCE_HOUSEHOLD_ID_COLUMN].tolist(), + household[_CLONE].tolist(), + strict=True, + ): + path = ((GEOGRAPHIC_SUPPORT_BRANCH, clone),) if clone else () + assert key == household_draw_key( + source=GEOGRAPHIC_IDENTITY_SOURCE, + source_vintage=_RELEASE.vintage, + source_household_id=source, + clone_path=path, + ) + + +def test_keys_are_unique_and_a_larger_k_keeps_every_key() -> None: + small = _keys(key_uk_households_by_clone_path(_expanded(3)).frame) + large = _keys(key_uk_households_by_clone_path(_expanded(7)).frame) + assert len(set(large)) == len(large) + # Clone-major: the first three clones of seven are the three clones. + assert large[: len(small)] == small + + +def test_the_key_is_the_graphs_string_dtype_and_the_source_key_stays_object() -> None: + keyed = key_uk_households_by_clone_path(_expanded(3)).frame + household = keyed.table("household") + assert household[IDENTITY_COLUMN].dtype == dtype_for_token("string") + assert household[IDENTITY_COLUMN].dtype == pd.StringDtype(storage="python") + # The legacy audit's rule: object columns holding text. The source key + # is one, as the legacy plan's SPI support channel writes it; the + # geography key is not. + object_strings = { + column + for column in household.columns + if household[column].dtype == object + and household[column].map(lambda value: isinstance(value, str)).all() + } + assert SOURCE_HOUSEHOLD_KEY_COLUMN in object_strings + assert IDENTITY_COLUMN not in object_strings + + +def test_the_step_adds_the_key_alone() -> None: + expanded = _expanded(3) + keyed = key_uk_households_by_clone_path(expanded).frame + for entity in expanded.entities: + before = expanded.table(entity) + pd.testing.assert_frame_equal( + keyed.table(entity)[list(before.columns)], before, check_exact=True + ) + assert list(keyed.table("household").columns) == [ + *expanded.table("household").columns, + IDENTITY_COLUMN, + ] + assert ( + keyed.weights_for("household").values.tobytes() + == expanded.weights_for("household").values.tobytes() + ) + assert keyed.weights_for("household").kind is expanded.weights_for("household").kind + assert keyed.mass_log == expanded.mass_log + assert dict(keyed.metadata) == dict(expanded.metadata) + + +def test_the_vintage_is_frs_release_json_s_for_the_source_year() -> None: + assert dict(frs_vintages_by_source_year()) == { + _RELEASE.survey_year: _RELEASE.vintage, + _RELEASE.base_year: _RELEASE.vintage, + } + # Another pin, for the year before, keys a frame of that year with its + # own vintage. + earlier = replace( + _RELEASE, + survey_year=_RELEASE.survey_year - 1, + vintage="invented_vintage", + ) + expanded = _expanded(2, time_period=str(_RELEASE.survey_year - 1)) + result = key_uk_households_by_clone_path(expanded, release=earlier) + household = expanded.table("household") + assert _keys(result.frame) == [ + household_draw_key( + source=GEOGRAPHIC_IDENTITY_SOURCE, + source_vintage="invented_vintage", + source_household_id=source, + clone_path=((GEOGRAPHIC_SUPPORT_BRANCH, clone),) if clone else (), + ) + for source, clone in zip( + household[SOURCE_HOUSEHOLD_ID_COLUMN].tolist(), + household[_CLONE].tolist(), + strict=True, + ) + ] + assert result.evidence()["source_vintages"] == ["invented_vintage"] + + +def test_a_base_year_after_the_survey_year_keys_with_the_release_s_vintage() -> None: + """G1 writes source_year from the frame's time period, the release's base + year, so a pin whose base year follows its survey year keys the base + year's households with its vintage, as the dense identity does.""" + + later = replace( + _RELEASE, + base_year=_RELEASE.survey_year + 1, + time_period=str(_RELEASE.survey_year + 1), + ) + assert dict(frs_vintages_by_source_year(later)) == { + _RELEASE.survey_year: _RELEASE.vintage, + _RELEASE.survey_year + 1: _RELEASE.vintage, + } + expanded = _expanded(3, time_period=later.time_period) + result = key_uk_households_by_clone_path(expanded, release=later) + assert _keys(result.frame) == _dense_keys(expanded.table("household")) + assert result.evidence()["source_vintages"] == [_RELEASE.vintage] + + +def test_the_source_is_the_dense_identity_s() -> None: + parameters = inspect.signature(append_uk_population_nodes).parameters + assert parameters["identity_source"].default == GEOGRAPHIC_IDENTITY_SOURCE + + +def test_the_evidence_gives_rounded_sources_uniqueness_source_and_vintage() -> None: + result = key_uk_households_by_clone_path(_expanded(3, households=9)) + assert result.evidence() == { + "stage": GEOGRAPHIC_IDENTITY_STAGE_NAME, + # Distinct source households, a household's clones counted once: + # the same at every K. + "source_households": rounded_count(9), + "unique": True, + "source": GEOGRAPHIC_IDENTITY_SOURCE, + "source_vintages": [_RELEASE.vintage], + "inputs": [ + SOURCE_HOUSEHOLD_ID_COLUMN, + SOURCE_YEAR_COLUMN, + ladder_clone_index_column("household"), + ], + } + assert list(GEOGRAPHIC_IDENTITY_INPUTS) == result.evidence()["inputs"] + assert ( + key_uk_households_by_clone_path(_expanded(1, households=9)).evidence() + == result.evidence() + ) + small = key_uk_households_by_clone_path(_expanded(3, households=2)) + assert small.evidence()["source_households"] == "<3" + # The result's repr prints no count and no key. + assert repr(small) == ( + f"UKGeographicIdentityResult(source_vintages=('{_RELEASE.vintage}',))" + ) + + +def test_the_transform_runs_the_step_and_records_its_evidence() -> None: + expanded = _expanded(3) + transform = UKGeographicIdentityStageTransform(_stage()) + with pytest.raises(RuntimeError, match="completed stage run"): + transform.checkpoint_metadata() + keyed = transform(expanded) + direct = key_uk_households_by_clone_path(expanded) + pd.testing.assert_frame_equal( + keyed.table("household"), direct.frame.table("household"), check_exact=True + ) + assert transform.checkpoint_metadata() == {"evidence": direct.evidence()} + assert transform.output_columns() == (IDENTITY_COLUMN,) + + +# --- refusals --------------------------------------------------------------------- + + +def _assert_no_key_or_id(message: str, frame: Frame) -> None: + assert "uk-household-v1" not in message + for household_id in frame.table("household")["household_id"].tolist(): + if household_id > 1000: + assert str(household_id) not in message + + +def test_a_year_the_release_pins_no_vintage_for_is_refused_by_count() -> None: + other = _expanded(2, time_period=str(_UNPINNED_EARLIER)) + with pytest.raises( + ValueError, match="^18 household\\(s\\) carry a source_year that frs_release" + ): + key_uk_households_by_clone_path(other) + expanded = _expanded(3) + household = expanded.table("household").copy() + household.loc[household.index[[1, 4, 7, 20]], SOURCE_YEAR_COLUMN] = _UNPINNED_LATER + with pytest.raises( + ValueError, match="^4 household\\(s\\) carry a source_year" + ) as error: + key_uk_households_by_clone_path(_with_household(expanded, household)) + _assert_no_key_or_id(str(error.value), expanded) + + +def test_repeated_keys_are_refused_by_count() -> None: + expanded = _expanded(2) + household = expanded.table("household").copy() + # Clone 1 loses its clone index: its keys repeat clone 0's. + household[_CLONE] = np.zeros(len(household), dtype=np.int64) + with pytest.raises( + ValueError, match="^9 household\\(s\\) repeat an earlier household's geography" + ) as error: + key_uk_households_by_clone_path(_with_household(expanded, household)) + _assert_no_key_or_id(str(error.value), expanded) + + +@pytest.mark.parametrize( + ("column", "message"), + [ + (_CLONE, r"lacks the 'geographic_support' step's column\(s\)"), + (SOURCE_YEAR_COLUMN, "reads household 'source_year', which the geographic"), + ( + SOURCE_HOUSEHOLD_ID_COLUMN, + "reads household 'source_household_id', which the geographic", + ), + ], +) +def test_a_frame_without_g1_s_columns_is_refused(column: str, message: str) -> None: + expanded = _expanded(2) + household = expanded.table("household").drop(columns=[column]) + with pytest.raises(ValueError, match=message) as error: + key_uk_households_by_clone_path(_with_household(expanded, household)) + _assert_no_key_or_id(str(error.value), expanded) + + +@pytest.mark.parametrize("column", [SOURCE_HOUSEHOLD_ID_COLUMN, SOURCE_YEAR_COLUMN]) +def test_a_source_column_that_is_not_int64_is_refused(column: str) -> None: + expanded = _expanded(2) + household = expanded.table("household").copy() + household[column] = household[column].astype(np.int32) + with pytest.raises(ValueError, match=f"reads an int64 '{column}'; it holds int32"): + key_uk_households_by_clone_path(_with_household(expanded, household)) + + +def test_a_frame_already_keyed_is_refused() -> None: + keyed = key_uk_households_by_clone_path(_expanded(2)).frame + with pytest.raises(ValueError, match="already carries 'geography_household_key'"): + key_uk_households_by_clone_path(keyed) + + +def test_a_non_frame_is_refused() -> None: + with pytest.raises(TypeError, match="keys a Frame"): + key_uk_households_by_clone_path(_expanded(2).table("household")) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"stage": "geographic_expand"}, "Expected stage 'geographic_identity'"), + ({"grain": "person"}, "grain must be 'household'"), + ({"outputs": ()}, "must declare outputs"), + ({"rewrites": ("region",)}, "and no rewrites"), + ], +) +def test_a_stage_that_is_not_g1b_s_is_refused( + overrides: dict[str, object], message: str +) -> None: + with pytest.raises(ValueError, match=message): + UKGeographicIdentityStageTransform(_stage(**overrides)) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 47c171c3cb..a650e5f412 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -295,7 +295,11 @@ def test_every_stage_that_counts_by_lineage_binds_the_counting_modules() -> None def test_counting_module_changes_invalidate_every_stage_that_counts(monkeypatch): from microcosm.build import source_lineage - from microcosm.build.uk_runtime import graph_kernels, receipt_counts + from microcosm.build.uk_runtime import ( + geographic_clones, + graph_kernels, + receipt_counts, + ) graph = uk_spine_graph() registry = uk_registry(graph=graph) @@ -308,7 +312,7 @@ def test_counting_module_changes_invalidate_every_stage_that_counts(monkeypatch) for stage in sorted(graph_kernels._COUNTING_STAGES) ] control = graph.node("frs_employment").kernel - for module in (receipt_counts, source_lineage): + for module in (receipt_counts, source_lineage, geographic_clones): before = { ref: registry.implementation_hash(ref) for ref in (*affected, control) } @@ -1223,11 +1227,6 @@ def changed_module_bytes(path): #: can be listed (``_helper_kind``); the tests hold the table to exactly what #: they find, so a new one is bound or listed here with its reason. _STAGE_UNBOUND_NAMES: dict[tuple[str, str], str] = { - ("microcosm.build.uk_runtime.rowwise_dataset", "ladder_clone_index_column"): ( - "names the geographic clone index columns, which only the dense " - "expand writes; no spine stage writes them (checked), so a spine " - "frame never carries them and their name moves no spine value" - ), ("microcosm.frame.concepts", "SOURCE_HOUSEHOLD_ID_COLUMN"): ( "names the source household column, CopyLineage's default source, " "which the support channel writes by its literal name. A node is " @@ -1250,9 +1249,9 @@ def changed_module_bytes(path): ) for name in ("LAD23_NAMES_SHA256", "LAD23_NAMES_URL") }, - ("microcosm.build.uk_runtime.rowwise_dataset", "ladder_clone_index_column"): ( + ("microcosm.build.uk_runtime.geographic_clones", "ladder_clone_index_column"): ( "names the geographic clone index columns, which only the dense expand " - "writes, and uk.full.expand@1 hashes rowwise_dataset. The calibrated " + "writes, and uk.full.expand@1 hashes geographic_clones. The calibrated " "gates' support-family fold reads the household's on the expanded " "frame, so their key chains on the expand's; the national gates fold " "the national frame, which no ladder expands, and the fold keys on the " @@ -1564,7 +1563,7 @@ def test_every_name_a_stage_reads_is_hashed_with_it_upstream_or_listed() -> None stage's node or by a node upstream of it, whose key the stage's key chains on, or the name is listed in ``_STAGE_UNBOUND_NAMES`` with its reason; the table lists nothing else (microcosm#931).""" - from microcosm.build.uk_runtime import graph_kernels, rowwise_dataset + from microcosm.build.uk_runtime import graph_kernels from microcosm.graph.decl import compile_graph graph = uk_spine_graph() @@ -1598,15 +1597,6 @@ def upstream(node_id: str) -> set[str]: assert set(unlisted) == set(_STAGE_UNBOUND_NAMES), sorted( set(unlisted) ^ set(_STAGE_UNBOUND_NAMES) ) - # The geographic clone index columns' reason: no spine node writes them. - written = { - (owned.entity, owned.column) for node in graph.nodes for owned in node.outputs - } - for entity in ("person", "benunit", "household"): - assert ( - entity, - rowwise_dataset.ladder_clone_index_column(entity), - ) not in written # The source household column's reason. The support channel writes it by # its literal name; a stage that reads it declares it by that name and # counts before the copies only beside a copy column, or declares neither @@ -2172,6 +2162,7 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: import inspect from microcosm.build.uk_runtime import ( + geographic_clones, graph_build, graph_national, graph_population, @@ -2193,6 +2184,18 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: # before they read the columns' name. population = inspect.signature(graph_targets.append_uk_target_nodes).parameters assert population["population"].default == "uk.full.expand" + # And the premise of their reason, that only the dense expand writes the + # clone index columns: no spine node writes them. Once the spine's + # geographic expansion does, the reason is rewritten with this check. + written = { + (owned.entity, owned.column) + for node in uk_spine_graph().nodes + for owned in node.outputs + } + for entity in ("person", "benunit", "household"): + assert (entity, geographic_clones.ladder_clone_index_column(entity)) not in ( + written + ) def calls(module, name: str) -> list: tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_salary_sacrifice.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_salary_sacrifice.py index 3e1397cf1c..9c017b3995 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_salary_sacrifice.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_salary_sacrifice.py @@ -14,6 +14,7 @@ from microcosm.build.uk_runtime import salary_sacrifice from microcosm.build.uk_runtime.cgt_structure import HOUSEHOLD_IS_CGT_CLONE from microcosm.build.uk_runtime.cgt_support import HOUSEHOLD_IS_CGT_SUPPORT_COPY +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.salary_sacrifice import ( BONUS_SACRIFICE_OUTPUT, @@ -28,10 +29,15 @@ _assert_salary_sacrifice_stage_parameters, bonus_sacrifice_excess, convert_bonuses_to_bonus_sacrifice, + draw_spi_bonuses, impute_salary_sacrifice, load_bonus_sacrifice_anchor, load_salary_sacrifice_anchor, ) +from test_support.microcosm_build.uk_clone_first_frames import ( + GEOGRAPHIC_RADIX, + clone_first_frame, +) from test_support.microcosm_build.uk_geographic_clones import with_geographic_clones @@ -778,8 +784,6 @@ def test_a_bonus_is_sacrificed_whole_up_to_half_the_pay() -> None: def _spi_draw(**kwargs): - from microcosm.build.uk_runtime.salary_sacrifice import draw_spi_bonuses - n_donors = 25 defaults = dict( # 25 FRS donors at 100,000 with a tenth of pay as bonus, then two @@ -967,3 +971,191 @@ def test_geographic_clones_train_each_respondent_once_at_their_whole_weight( impute_salary_sacrifice(with_geographic_clones(frame)) alone, cloned = _FakeQRF.training_frames pd.testing.assert_frame_equal(cloned, alone) + + +#: Invented donors for the per-clone bonus draws: 24 base-FRS adults paid at +#: each of two levels, so two pay cells of 24 donor households each, with +#: bonus shares of pay from 0.05 up in steps of 0.004. +_DONOR_PAY = np.repeat([60_000.0, 150_000.0], 24) +_DONOR_SHARES = 0.05 + 0.004 * np.arange(len(_DONOR_PAY)) +#: Five SPI-channel adults: two in each pay cell and one without pay. +_SPI_PAY = np.asarray([80_000.0, 90_000.0, 300_000.0, 400_000.0, 0.0]) +#: How far each later clone's donor shares sit above clone 0's, so that every +#: clone's shares are its own. +_CLONE_SHARE_STEP = 0.0013 + + +def _clone_draw_inputs(n_clones: int) -> dict[str, np.ndarray]: + """The SPI bonus draw's inputs at ``n_clones`` geographic clones. + + Clone-major, as the clone-first spine orders them: clone ``k`` repeats + the one-clone rows with ids up ``k x 10**9``, the same source + households, ``1 / n_clones`` of each weight and donor shares + ``k x _CLONE_SHARE_STEP`` higher. Clone 0 is the one-clone draw's rows + at ``w / n_clones``. + """ + + n_donors, n_spi = len(_DONOR_PAY), len(_SPI_PAY) + ids = np.arange(1, n_donors + n_spi + 1, dtype=np.int64) + rows = np.arange(len(ids)) + weights = 1.0 + 0.5 * (ids % 4) + + def clone(k: int) -> dict[str, np.ndarray]: + return { + "bonus": np.concatenate( + [ + _DONOR_PAY * (_DONOR_SHARES + _CLONE_SHARE_STEP * k), + np.full(n_spi, 99_000.0), + ] + ), + "employment_income": np.concatenate([_DONOR_PAY, _SPI_PAY]), + "weights": weights / n_clones, + "person_ids": ids + k * GEOGRAPHIC_RADIX, + "source_ids": ids, + "donors": rows < n_donors, + "spi": rows >= n_donors, + "clone_indices": np.full(len(ids), k, dtype=np.int64), + } + + clones = [clone(k) for k in range(n_clones)] + return {name: np.concatenate([c[name] for c in clones]) for name in clones[0]} + + +def _drawn_donor_rows(draw, inputs: dict[str, np.ndarray]) -> dict[int, int]: + """For each SPI adult with a drawn bonus, the donor row whose share it took. + + Every donor share differs from every other by at least + ``_CLONE_SHARE_STEP``, so the nearest share names the donor. + """ + + pay = inputs["employment_income"] + donors = np.flatnonzero(inputs["donors"]) + shares = np.minimum(inputs["bonus"][donors] / pay[donors], 1.0) + return { + int(row): int(donors[np.argmin(np.abs(shares - draw.bonus[row] / pay[row]))]) + for row in np.flatnonzero(draw.recipients) + } + + +def test_each_spi_adult_draws_a_bonus_share_from_its_own_clones_donors() -> None: + """Geographic clones of a spine draw SPI bonuses within each clone + (microcosm#931). The pay cells and the donor-source floor pool every + clone's donors, counted in source households, so three clones keep the + one-clone cells; each SPI adult takes a donor of its own clone in its + cell; and clone 0, whose ids are the one-clone ids, draws as one clone + does at a third of the weight.""" + + one = _clone_draw_inputs(1) + del one["clone_indices"] + alone = draw_spi_bonuses(**one) + # A one-clone index draws as no index, value for value. + indexed = draw_spi_bonuses( + **one, clone_indices=np.zeros(len(one["spi"]), dtype=np.int64) + ) + assert indexed.bonus.tobytes() == alone.bonus.tobytes() + assert indexed.cells == alone.cells == 2 + + three = _clone_draw_inputs(3) + cloned = draw_spi_bonuses(**three) + assert cloned.cells == alone.cells + assert cloned.bonus[: len(alone.bonus)].tobytes() == alone.bonus.tobytes() + clone, pay = three["clone_indices"], three["employment_income"] + drawn = _drawn_donor_rows(cloned, three) + # Two SPI adults with pay in each cell, in each of the three clones. + assert len(drawn) == 12 + for row, donor in drawn.items(): + assert clone[donor] == clone[row] + assert (pay[donor] == _DONOR_PAY.min()) == (pay[row] < _DONOR_PAY.max()) + + +def test_a_cell_with_no_donor_in_an_spi_adults_clone_is_refused() -> None: + """The refusal names no count, id or value.""" + + inputs = _clone_draw_inputs(2) + # Clone 1's donors carry no weight, so its SPI adults find no donor in + # their cells, though clone 0's donors fill both cells' floor. + second_clone_donors = inputs["donors"] & (inputs["clone_indices"] == 1) + inputs["weights"] = np.where(second_clone_donors, 0.0, inputs["weights"]) + with pytest.raises( + ValueError, match="no donor to draw in an SPI adult's" + ) as refused: + draw_spi_bonuses(**inputs) + assert not any(character.isdigit() for character in str(refused.value)) + + +def _bonus_clone_frame(): + """The draw's invented donors and SPI adults as a one-clone stage frame. + + Everyone is asked and nobody contributes to a pension, so the forest + predicts for no one and the salary conversion moves no pay. + """ + + pay = np.concatenate([_DONOR_PAY, _SPI_PAY]) + n = len(pay) + return _frame( + asked=[1] * n, + salary_sacrifice_values=[0.0] * n, + employee_pension=[0.0] * n, + channels=["frs"] * len(_DONOR_PAY) + ["spi"] * len(_SPI_PAY), + weights=1.0 + 0.5 * (np.arange(1, n + 1) % 4), + employment_income=pay, + bonus=np.concatenate( + [_DONOR_PAY * _DONOR_SHARES, np.full(len(_SPI_PAY), 99_000.0)] + ), + ) + + +def test_the_stage_draws_each_clones_spi_bonuses_from_that_clone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The stage reads a person's geographic clone from its household's + ``household_clone_index`` (microcosm#931). A one-clone index changes + nothing; at three clones, clone 0 sacrifices the bonuses one clone + sacrifices, and every SPI adult's bonus is a share of a donor in its own + clone.""" + + _Fitted.value = 0.0 + monkeypatch.setattr(salary_sacrifice, "QRF", _FakeQRF) + # Every candidate converts, and each drawn bonus is under half its pay, + # so an SPI adult sacrifices the bonus it drew. + anchor = _toy_bonus_anchor(obr_base_above_threshold=1e12) + frame = _bonus_clone_frame() + alone = impute_salary_sacrifice(frame, bonus_anchor=anchor) + indexed = impute_salary_sacrifice(clone_first_frame(frame, 1), bonus_anchor=anchor) + pd.testing.assert_frame_equal( + indexed.frame.table("person"), alone.frame.table("person"), check_exact=True + ) + assert indexed.evidence() == alone.evidence() + + three = clone_first_frame(frame, 3) + household = three.table("household").set_index("household_id") + person = three.table("person").copy() + members = person["person_household_id"] + clone = members.map(household[ladder_clone_index_column("household")]).to_numpy() + donors = members.map(household["household_support_channel"]).eq("frs").to_numpy() + pay = person["employment_income"].to_numpy() + # Each later clone's donors give bonus shares of their own. + person[BONUS_SACRIFICE_SOURCE] += np.where( + donors, _CLONE_SHARE_STEP * clone * pay, 0.0 + ) + cloned = impute_salary_sacrifice( + uk_national_frame( + person=person, + benunit=three.table("benunit"), + household=three.table("household"), + household_weights=three.weights_for("household").values, + weight_kind=three.weights_for("household").kind, + time_period="2024", + ), + bonus_anchor=anchor, + ) + sacrificed = cloned.frame.table("person")[BONUS_SACRIFICE_OUTPUT].to_numpy() + one_clone = alone.frame.table("person")[BONUS_SACRIFICE_OUTPUT].to_numpy() + assert sacrificed[: len(one_clone)].tobytes() == one_clone.tobytes() + donor_rows = np.flatnonzero(donors) + shares = person[BONUS_SACRIFICE_SOURCE].to_numpy()[donor_rows] / pay[donor_rows] + takers = np.flatnonzero(~donors & (pay > 0.0)) + assert len(takers) == 12 + for row in takers: + donor = donor_rows[np.argmin(np.abs(shares - sacrificed[row] / pay[row]))] + assert clone[donor] == clone[row] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py index 3af4460f5e..74f12d6ab3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py @@ -1,7 +1,8 @@ """Tests split from packages/microcosm-build/tests/test_uk_uc_capital_coherence.py.""" # ruff: noqa: F403, F405 -from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame from test_support.microcosm_build.uk_geographic_clones import with_geographic_clones from test_support.microcosm_build.uk_uc_capital_coherence import * @@ -636,9 +637,9 @@ def test_stage_refuses_a_drifted_property_share_declaration() -> None: def test_geographic_clones_donate_once_at_their_whole_weight() -> None: """Two geographic clones of every unit at half its weight donate as the - units donate alone: every SPI unit's first clone draws the capital it - draws without clones, and a cell's donor count reads units, not clones, - so each clone draws at its unit's coarsening level (microcosm#931).""" + units donate alone: every SPI unit's clone 0 draws the capital it draws + without clones, and a cell's donor count reads units, not clones, so + each clone draws at its unit's coarsening level (microcosm#931).""" alone = cohere_uc_capital(_frame(), minimum_cell_donors=2) cloned = cohere_uc_capital(with_geographic_clones(_frame()), minimum_cell_donors=2) @@ -651,3 +652,120 @@ def test_geographic_clones_donate_once_at_their_whole_weight() -> None: assert cloned.coarsening_levels == { level: 2 * count for level, count in alone.coarsening_levels.items() } + + +def _rebuilt( + frame, *, person=None, benunit=None, household=None, household_weights=None +): + """``frame`` with the given tables or weights replaced, at its weight kind.""" + + weights = frame.weights_for("household") + return uk_national_frame( + person=frame.table("person") if person is None else person, + benunit=frame.table("benunit") if benunit is None else benunit, + household=frame.table("household") if household is None else household, + household_weights=( + weights.values if household_weights is None else household_weights + ), + weight_kind=weights.kind, + time_period="2024", + ) + + +def _variant_clones(frame, n_clones: int): + """``frame`` as ``n_clones`` geographic clones whose base units differ. + + Each base unit's capital rises by one pound a clone, so no donor's clones + agree on what it donates, and every clone donates as a variant of its own. + """ + + cloned = clone_first_frame(frame, n_clones) + person = cloned.table("person") + unit_households = person.drop_duplicates("person_benunit_id").set_index( + "person_benunit_id" + )["person_household_id"] + household_clones = cloned.table("household").set_index("household_id")[ + ladder_clone_index_column("household") + ] + benunit = cloned.table("benunit").copy() + clone = benunit["benunit_id"].map(unit_households).map(household_clones) + base = benunit[support_channel_column("benunit")].eq("frs") + benunit["frs_benunit_capital"] += np.where(base, clone, 0) + return _rebuilt(cloned, benunit=benunit) + + +def _with_a_high_value_copy(frame, household_id: int, *, offset: int): + """``frame`` with a high-value support copy of one household. + + The copy keeps its root's source household and every value, with each + id up ``offset`` and the high-value copy flag and index set. + """ + + person = frame.table("person") + benunit = frame.table("benunit") + household = frame.table("household") + root = household["household_id"].eq(household_id).to_numpy() + members = person.loc[person["person_household_id"].eq(household_id)].copy() + units = benunit.loc[benunit["benunit_id"].isin(members["person_benunit_id"])] + units = units.copy() + copy = household.loc[root].copy() + for column in ("person_id", "person_benunit_id", "person_household_id"): + members[column] += offset + units["benunit_id"] += offset + copy["household_id"] += offset + copy["household_is_high_value_support_copy"] = True + copy["high_value_support_copy_index"] = 1 + weights = frame.weights_for("household").values + return _rebuilt( + frame, + person=pd.concat([person, members], ignore_index=True), + benunit=pd.concat([benunit, units], ignore_index=True), + household=pd.concat([household, copy], ignore_index=True), + household_weights=np.concatenate([weights, weights[root]]), + ) + + +def test_a_one_clone_index_redraws_as_a_frame_without_clones() -> None: + """At one clone every donor row is a survey record of its own, so each + cell counts its donor rows, as before (microcosm#931).""" + + for minimum in (1, 2, 3): + alone = cohere_uc_capital(_frame(), minimum_cell_donors=minimum) + indexed = cohere_uc_capital( + clone_first_frame(_frame(), 1), minimum_cell_donors=minimum + ) + for entity in ("person", "benunit"): + pd.testing.assert_frame_equal( + indexed.frame.table(entity), alone.frame.table(entity), check_exact=True + ) + assert indexed.evidence() == alone.evidence() + + +def test_variant_clones_count_once_toward_a_cells_donors() -> None: + """A unit's geographic clones that differ donate apart, yet count as one + survey record toward the donors a cell needs, so at three clones every + SPI unit's clones draw at its one-clone coarsening level (María's ruling + of 2026-11-10, microcosm#931). Counted in rows, the couple reporter cell's + one donor would count three and stop widening there.""" + + alone = cohere_uc_capital(_frame(), minimum_cell_donors=2) + cloned = cohere_uc_capital(_variant_clones(_frame(), 3), minimum_cell_donors=2) + # The couple reporter unit's exact cell holds one donor, so it widens. + assert alone.coarsening_levels[UC_CAPITAL_COARSENING[1]] == 1 + assert cloned.coarsening_levels == { + level: 3 * count for level, count in alone.coarsening_levels.items() + } + + +def test_a_high_value_copy_counts_as_a_donor_of_its_own() -> None: + """A copy other than a geographic clone is a survey record of its own, + though it keeps its root's source household (María's ruling of + 2026-11-10, microcosm#931): a high-value copy of the couple reporter + donor gives that cell the two donors it needs.""" + + alone = cohere_uc_capital(_frame(), minimum_cell_donors=2) + copied = cohere_uc_capital( + _with_a_high_value_copy(_frame(), 3, offset=10_000), minimum_cell_donors=2 + ) + assert alone.coarsening_levels["exact_cell"] == 2 + assert copied.coarsening_levels["exact_cell"] == 3 diff --git a/test_support/microcosm_build/uk_cgt_support.py b/test_support/microcosm_build/uk_cgt_support.py index b521f5a28e..9dbf8d6aea 100644 --- a/test_support/microcosm_build/uk_cgt_support.py +++ b/test_support/microcosm_build/uk_cgt_support.py @@ -61,6 +61,32 @@ 200_000: 250_000.0, } +#: The clone-first nesting scenario (microcosm#931), which +#: test_uk_cgt_support.py and test_uk_clone_first_nesting.py split at several +#: clone counts: nine one-person households, their weights, investable +#: wealth and Table 3 incomes (seven in band 0, one in band 37,700, then the +#: zero-weight one in band 0), and a joint that asks band 0 for far more than +#: its pool and band 37,700 for nothing. test_uk_cgt_support.py says where +#: rule B's correction lands at each clone count, and why. +NESTING_WEIGHTS: tuple[float, ...] = ( + 900.0, + 1_800.0, + 4_020.0, + 7_620.0, + float(np.nextafter(300.0, 0.0)), + 40.0, + 1_234.5, + 9_000.0, + 0.0, +) +NESTING_WEALTH: tuple[float, ...] = (7e3, 6e3, 5e3, 4e3, 3e3, 2e3, 1e3, 1.0, 1e6) +NESTING_INCOMES: tuple[float, ...] = ( + *[BAND_INCOMES[0]] * 7, + BAND_INCOMES[37_700], + BAND_INCOMES[0], +) +NESTING_TOP_CELLS: Mapping[int, Mapping[int, float]] = {0: {250_000: 1_000_000.0}} + def support_distribution( top_cells: Mapping[int, Mapping[int, float | None]] | None = None, From a7586b5dd3ef2363620f324eeb193383440f570f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:53:08 +0000 Subject: [PATCH 059/108] Clone the UK spine after its source stages: move stages 14 and 15 up and add geographic_expand and geographic_identity (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit frs_hmrc_spine_leaves and frs_annual_income_spells (stages 14 and 15) move ahead of frs_take_up, and two new stages follow them: geographic_expand (G1) gives every FRS household K geographic clones, and geographic_identity (G1b) keys each clone. frs_take_up, frs_person_draws, frs_household_draws and frs_brma (stages 10 to 13) then run on every clone. The spine build installs both new stages at K = 1, so every spine is still one clone and no value moves. Positional stage numbers in this message, the changelog, comments and tests follow the order before this commit: 14 and 15 are the two moved stages, 10 to 13 the draws and frs_brma, and 16 onward are as before. The order. sources.yaml moves the two stage blocks unchanged and inserts G1 and G1b after them; the graph follows the manifest. The moved stages read only pay, self-employment income, age and gender; stages 10 to 12 read none of their outputs and write none of their inputs, and none of the five sets weights (a test holds this). Both moved stages join the raw FRS tabs on raw person ids, so they run on the survey's own rows, before the clones. The assembled checkpoint still ends at frs_brma, so the assembled phase and its gates now cover 14, 15, G1, G1b and 10 to 13, and the assembled gate admits spi_support_channel. G1 and G1b, declared and wired at K = 1: - The sources schema and ALLOWED_SOURCE_OPERATION_KINDS gain two strict op kinds. expand_geographic_clones declares the entity, the clone-index columns, the ordinal bound of 100, the copy-id geographic_support field at radix 10**9, clone-major order, an equal design-weight split, the source columns, conservation to the split's rounding, declared factor 1.0 and a reason that names no K. key_households_by_clone_path declares the source frs and frs_release.json, which G1b also declares as its one artifact with runtime_sha256_required, so the release's bytes enter its stage contract. Each transform checks its declaration against its module's reviewed parameters, whole; G1 refuses any artifact and G1b any but that one. - No declaration names K. uk_spine_graph(geographic_clones=K) puts K, 1 to GEOGRAPHIC_CLONE_LIMIT, on G1's node params, and refuses K above 1 for a manifest without G1, so K enters that node's key and every later one, never a stage contract. - graph.py: G1 joins UK_SPINE_STRUCTURAL_STAGES as an EXPAND that conserves mass at design weights, with an open surface and six cells. Its household cells are listed in the order the transform writes them (source_year, source_household_key, source_household_id, household_clone_index), since the graph appends an EXPAND's new cells in that order and H2 parity compares column order; an engine-free test now holds that order too. G1b reads the household source id, source year and clone index and writes geography_household_key. uk.full.identity rewrites the key where the spine already carries it, so the dense graph still compiles. - graph_kernels.py: both stages get their modules and the helpers the derived rule finds (G1: clone_expansion and copy_id_table; G1b: atomic_household_identity, frs_release and microcosm.graph.canonical) and join the counting stages, and G1 joins the copy-id readers at geographic_support. The registry's structural set is UK_SPINE_STRUCTURAL_STAGES. UKExpandStageKernel refuses a transform whose clone count differs from its node's, giving the two declared counts. The fixture registry builds G1 and G1b from an integer config.geographic_clones and its engine with clones=K, and its descriptor check no longer names a stage count. - spine_build: _GEOGRAPHIC_CLONES = 1 feeds the graph, the G1 transform and _rules_engine(clones=1). Its clones keyword defaults to None, so the full build's two engine installs are unchanged. G1 and G1b are installed when the roster names them. - The geography key's name, GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, now lives in the geographic_clones leaf, which G1b hashes as a counting stage, and atomic_area_support re-exports it as IDENTITY_COLUMN, so the derived rule holds every reader to the leaf and the stage table needs no entry for it; no kernel that reads it misses the leaf. G1b names its string dtype through pandas, as atomic_household_lineage does, not through the executor's token table. The identity columns. source_household_id, source_year and source_household_key are G1's outputs and cells. spi_support_channel (stage 16) no longer declares or writes them, and its copies inherit them. The stage and the dense expansion default them from a household's own ids only on a table without household_clone_index, and refuse a clone-indexed table that lacks any of them, naming the columns. source_person_id and source_benunit_id are not minted (decision 5). Their names (UK_GEOGRAPHIC_SOURCE_COLUMNS, SOURCE_YEAR_COLUMN, SOURCE_HOUSEHOLD_KEY_COLUMN) move to the geographic_clones leaf, which spells the source id itself, since it imports nothing first-party (a test holds it equal to microcosm.frame.concepts'); geographic_expand re-exports them, and the SPI support channel and the dense crosswalk read the leaf's tuple instead of copies of their own. Stages 10 to 12 count true sources. frs_take_up, frs_person_draws and frs_household_draws are given household source_household_id and household_clone_index, so the shared reader counts their receipts' support in source households on every clone; it refuses the clone index without the source column. A test holds every stage after G1 that counts before the copies to both columns. Folding consumes. household_clone_index joins the inputs of spi_employment_shell, spi_housing_shell and uc_capital_coherence, which fold geographic clones, and of property_components. The typed root cells. frs_spine writes imd_decile (int64: the household's decile on its own nation's FRS index, IMDE, IMDW, IMDS or IMDN, 1 to 10, else 0, since the root refuses NaN) and council_tax_band_observed (bool: False in Northern Ireland, for a ctband code outside 1 to 9, for a band the region does not use and for an unknown region). council_tax_band itself is unchanged. The root refuses a household tab without one of the four IMD variables, naming it, rather than read zeros. Both cells are typed, leave at the release boundary (UK_RELEASE_EXPORT_DROPPED_COLUMNS) and stay out of every receipt and the sidecar's nonzero shares, whichever route built the records; the root's notes describe them. The root binds geography_constants, since both read each household's nation from the region tier. Their first reader is the area assignment of #931 group (c). microdf in node keys. microdf-python's version joins _numerical_dependency_versions ("not-installed" when absent), so it is in the root's and every stage's node params, and the sidecar records it as microdf_python beside engine_batches when a call batches; on that route private_school_attendance stores microdf's weighted percentile rank. pyproject.toml, uv.lock and us_runtime/worker_identity.py are unchanged; the version is declared at the next lock bump. Texts: - CGT_SUPPORT_EXACT_TOTAL_EXCEPTION, and with it the copy_rule of both support splits, high_value_home_support (stage 26) and cgt_support_split (stage 42), names the candidate order: heaviest first, tied weights in row order. - The notes of isa_subscriptions (stage 53) say tied balances share one weighted mid-rank, so the keyed uniform alone orders them, and the Lifetime ISA payments reuse the amounts operation's balance_rank_weight. docs/uk-isa-subscriptions-1415.md steps 3, 5 and 6 say the rank is on the class's balance, the added people walk on their own, holders over 47 pay nothing, and the cap at each holder's room keeps the baseline under the limit. - The weight rule of spi_income_band_donors (stage 17) names one clone's donor count and its draw a spine without household_clone_index; its notes size the seats on eligible households counted in source households. The notes of spi_support_channel (stage 16) say what its uniform depends on, and its source names every geographic clone. cgt_support_split's notes and the CGT changelog say which household the identity tool's recompute can count as a family the stage never made, and that the bracketed family test does not report it. - The source and notes of private_school_attendance (stage 52) say it computes attendance over the whole frame, from the pinned engine's values and parameters. - BONUS_SACRIFICE_SPI_DRAW and the notes of salary_sacrifice (stage 48) say an SPI adult draws within its own geographic clone from cells pooled over every clone; the draw text now also says the lowest cell joins the one above, as the code does. - uc_capital_coherence's notes count a cell's donors in survey records. - G1's notes say it clones before any other copy step, and childcare_copy_take_up's notes and docstring say frs_take_up draws each geographic clone on its own ids before any other copy exists, and that geographic clones keep those draws. - Texts this change made stale: household_lineage, spi_support, spi_band_donors, geographic_clones, geographic_expand, geographic_identity, salary_sacrifice, graph_population, diagnostics, copy_id_table, gate_phase_codes (six of the transferred-only modules are now hashed at or before frs_brma), receipt_counts, frs_take_up, private_school_attendance, frs_spine's and national_frame's band-flag wording (a band the FRS gives that the region uses), the clone-first stand-in, the identity tool's E7 docstring, docs/uk-full-build-graph.md (56 stages, the clone index on every spine frame, microdf_python in the sidecar), and the earlier #931 changelog fragments that said G1 was unwired, that no spine carries a clone index, or that the stage table and the kernel table read as before; and the new fragment 931-uk-spine-geographic-expand.changed.md. The code-mirrored texts are identical in the code, sources.yaml and source_stages.json. The was_wealth and was_lisa notes are unchanged. Bindings. The national gates hash geographic_clones, since their support-family fold reads the clone index columns, which the national frame carries from a clone-first spine; the kernel table's clone-index reason now covers only the calibrated gates, the dense measures and the national problem. The stage table keeps the source household column's name alone, its reason rewritten: geographic_expand writes it through the constant and declares it by name, every stage after it that reads the name is given the column, and frs_annual_income_spells, before it, is given neither the column nor a copy column. The kernel table's premise check now asserts that geographic_expand alone writes the clone-index columns on the spine and that the spine build installs it at one clone. Tools. tools/graph_uk_spine_fixture.py's oracle builds G1 and G1b from their packaged stages at _GEOGRAPHIC_CLONES = 1 and its engine with clones=1; its config records geographic_clones: 1, and uk_spine.json is written at that K. Its synthetic households gain IMDE, IMDW, IMDS and IMDN (invented deciles in turn, one household in seventeen blank). Its descriptor and PRODUCED_BY.txt no longer give a transform count. tools/build_uk_release_input_coverage_manifest.py gains a geographic_expand family, whose required reason is G1's declared reason, matched verbatim. Dense and national. The dense expansion refuses a spine carrying geographic_expand's clone index at any n_clones, naming the clone-first spine and #931 group (c), and its overflow message limits "n_clones=1 still expands" to a spine without the index. The national line is not yet refused early; a later commit of this group adds that refusal. No national or dense build runs on a clone-first spine before (c) (decision 4). Why: María's rulings of 10 Nov. Decision 3 (X) puts G1 and G1b after stages 14 and 15 and before stage 10, so the stages that read raw tabs by raw person id run on the survey's rows and stages 10 to 13 run on every clone from this commit. Decision 1 (Form B) puts stage 53's tie clause in its notes, which enter no stage contract. Decision 2 binds microdf's version into node keys without a re-lock, which would move the US approved-lock digest. Decision 5 mints no person or benefit unit source columns until something reads them. Every declaration edit of group (b) is in this commit, so H2 regenerates once. The clone columns' names sit in the geographic_clones leaf so that G1, G1b, the SPI support channel and the dense line read one definition their keys hash, rather than copies or a listed exemption. What moves. At K = 1 no value moves: on the H2 fixture the legacy oracle's final frame agrees with the parent's on every existing column, bit for bit and in dtype, on the weights and on every earlier mass record (checked through the engine slot). frs_take_up, frs_person_draws and frs_household_draws now count their receipts' support in true source households, which at one clone are the households themselves. The bytes move: - the stage order: frs_hmrc_spine_leaves and frs_annual_income_spells now run ahead of frs_take_up, with geographic_expand and geographic_identity between them; - the column order, since the households' source columns now come from geographic_expand; - each table's clone index, at 0; - geography_household_key, which a household's copies repeat until group (c); - the root's imd_decile and council_tax_band_observed, which the release drops and no receipt or sidecar share reports; - geographic_expand's mass record and both new stages' evidence; - spi_support_channel's strata, grouped by a constant clone index; - spi_income_band_donors' funding strata, which gain the clone index, so each label ends "|0"; - the national support-family fold, whose key columns gain the clone index. Seven stage contracts move: frs_spine (its outputs), spi_support_channel (its source text, outputs and nonnegative outputs), spi_income_band_donors (weight_rule and draw), high_value_home_support and cgt_support_split (copy_rule), salary_sacrifice (spi_bonus_draw) and private_school_attendance (its source text). geographic_expand's and geographic_identity's contracts are new. K is a parameter of geographic_expand's graph node, never of a contract. No contract hashes a stage's reads or notes. The source id and clone index that frs_take_up, frs_person_draws and frs_household_draws now read move only their nodes' inputs, and so does the clone index that spi_employment_shell, spi_housing_shell, uc_capital_coherence and property_components now read. The notes of isa_subscriptions (the tie clause, Form B), uc_capital_coherence, frs_spine, geographic_expand and childcare_copy_take_up move no contract. The dense line refuses every clone-first spine, and the national line's degenerate gate would refuse the constant indices. The coverage gate now requires geographic_expand's mass record, so a release from a spine built before this commit fails it too. No national or dense build runs between (b) and (c) (decision 4). Positional stage numbers in prose (comments, tests and changelogs) still follow the order before this commit. Hash and fixture. Every UK spine node re-keys: - microdf-python's version joins every node's numerical dependencies; - every stage kernel hashes graph_kernels and national_frame, both edited here; - the root's outputs move. sources.yaml, source_stages.json and the coverage manifest move the UK country fingerprint and the solve fingerprint, so every node keyed on one of them re-keys: the spine gates, the national targets and gates, the target selection, compilation, measures and problem, the full preflight and calibrated gates, and the rotated holdout. The rest of the national and dense lines re-key through the modules they hash: national_frame, graph_population, rowwise_dataset, geographic_clones, atomic_area_support, spi_support, household_lineage, receipt_counts and diagnostics, and the national gates through geographic_clones, which their key now hashes. Only the shared atomic-geography kernels and the export package keep their keys. Outside the UK one kernel re-keys: the shared transport gate battery (gates.battery@1). Its key covers the source closure of its gate bindings, which reaches source_manifest, so NZ's terminal gate and the export and package nodes after it re-key (the NZ spec is inactive). These keep their keys: - every other transport kernel; - the US post-transfer graph's nodes and kernels; - the US, NZ, BE and AM specs, seeds and fingerprints. uv.lock, pyproject.toml and the US approved-lock digest are untouched. The schema's two new operation kinds move only the spec engine's grammar receipt, which every country's emitted, uncommitted bundle lock carries. Regenerated: - source_stages.json, with the stage list merged by stage, so every other stage keeps its bytes and key order; - the release input coverage manifest, without --candidate-h5, then --check: a geographic_expand family, and source_stages.json's new digest in every family; - the H2 fixture, through the engine slot, then --check: uk_spine.json and sources/fixture.json (config.geographic_clones set to 1, both new stages and every declaration edit), household.csv and schema.json (the root's two carriers), frs_raw/househol.tab (synthetic IMDE, IMDW, IMDS and IMDN) and PRODUCED_BY.txt (no transform count). person.csv, benunit.csv, weights.json, NORMALIZATION.md and the donor and facts files regenerate byte for byte, and H2 parity passes. gates.json is unchanged. Tests (engine-free unless named; invented values only): - test_uk_graph.py: 56 stages; G1 in the EXPAND set and G1b a NONE node; eleven copy-id readers; the source household column's reason in the stage table; the geography key named in the leaf, read bound by G1b and re-exported by atomic_area_support; the kernel table's clone-index reason, whose premise is now that G1 alone writes the columns on the spine and the spine build installs it at one clone, and the national gates hashing the leaf; a stage after G1 that counts before the copies reads both lineage columns; every folding stage and property_components read the clone index; the moved stages and the draws read nothing the other writes; the root's typed carriers; G1 owns the source cells; microdf in every node's dependencies. - test_uk_geographic_expand.py and test_uk_geographic_identity.py build their transforms from the packaged stages and add: G1 is an EXPAND that conserves mass at design weights; K is on G1's node, never its contract; the graph's and the kernel's K refusals; no new row at K = 1 and tripled tables at K = 3 through the executor; the node lists each table's new cells in the order G1 writes them, at K = 1 and 3; the coverage family; the source columns named once in the leaf; the H2 registry building G1 and its engine at the fixture config's K, and refusing a K that is not an integer; a drifted declaration, a clones field and artifacts refused; G1b's node, and frs_release.json's bytes in its contract and not G1's. - test_uk_clone_first_nesting.py runs the packaged G1, and a K = 3 test runs stages 10 to 12 on the stand-in and holds their support counts to K = 1's. - test_uk_frs_spine.py: the carriers' types and codes, the absent-IMD refusal, codes that move only the carriers, the sidecar's share omission, and the band-flag test renamed for what the flag means; the driver's synthetic spec follows the packaged order with G1 and G1b, its engine stubs take clones, and the driver asks for its engine at one clone. - test_uk_spi_support.py and test_uk_spi_spine.py: the channel keeps G1's source columns and declares none; the clone-indexed refusal. - test_uk_full_population_graph.py: dense meets the reworded refusal at K = 1 and 2; the dense lineage refusal. test_uk_full_target_graph.py: uk.full.identity rewrites the key the spine carries. - test_uk_copy_id_table.py (G1's radix and the executor's trace at K = 1 and 3), test_uk_terminal_gates.py (the export drop), test_uk_rules_engine_provenance.py (microdf only when batching), test_uk_source_stages.py, test_uk_graph_evidence.py (the assembled gate admits spi_support_channel), test_uk_spine_acceptance_receipt.py, test_uk_identity_stability_receipts.py (G1 in the stacking-map exceptions), test_uk_spi_band_donors.py (frames through the real G1), test_uk_uc_capital_coherence.py, test_uk_cgt_support.py, test_uk_high_value_home_support.py and shared/test_country_spec.py: pins and literals. - Engine tier: tests/engine/uk/test_uk_graph.py subtracts the carriers from the nonzero-share pin, and test_uk_release_input_coverage.py's shipped-manifest pin gains geographic_expand. Checks. Before the review fixes, the whole of T/uk, run once in eight chunks, gave 7712 passed, 44 skipped and 2 failed; both failures fail at the parent too (one expects the UK engine absent, one needs a Node package this machine lacks). After them, every engine-free file they touch or that imports an edited module passes: 2652 passed and 14 skipped in 72 files, the graph package's engine-free H parity tests included. Through the engine slot, the fixture tool and its --check (byte-identical), H2 parity, tests/engine/uk/test_uk_graph.py and the shipped-manifest test pass. The source-stage regeneration and the coverage tool's --check report current, ruff check and ruff format --check are clean, and tools/ci_test_plan.py verify is ok. integration-uk and the rest of the engine-tier UK suite were not run here; this PR gets CI once #1519 merges. Co-Authored-By: Claude Opus 5.5 --- .../931-band-donors-per-clone.changed.md | 2 +- .../931-cgt-copies-source-weight.changed.md | 4 +- .../931-isa-tied-balance-rank.fixed.md | 2 +- changelog.d/931-keyed-tie-breaks.changed.md | 2 +- .../931-shared-clone-expansion-core.added.md | 2 +- ...31-uk-engine-blocks-pool-shares.changed.md | 2 +- ...31-uk-fits-fold-geographic-clones.added.md | 2 +- ...31-uk-floors-count-true-sources.changed.md | 2 +- changelog.d/931-uk-helper-bindings.changed.md | 6 +- .../931-uk-spine-geographic-expand.changed.md | 35 + .../931-uk-stages-ready-for-clones.changed.md | 7 +- docs/uk-full-build-graph.md | 4 +- docs/uk-isa-subscriptions-1415.md | 37 +- .../src/microcosm/build/source_manifest.py | 4 + .../spec_engine/schema/sources.schema.json | 85 ++ .../uk/release_input_coverage_manifest.json | 77 +- .../src/microcosm/build/uk/source_stages.json | 912 ++++++++++-------- .../src/microcosm/build/uk/spec/sources.yaml | 536 +++++----- .../build/uk_runtime/atomic_area_support.py | 9 +- .../microcosm/build/uk_runtime/cgt_support.py | 9 +- .../uk_runtime/childcare_copy_take_up.py | 9 +- .../build/uk_runtime/copy_id_table.py | 5 +- .../microcosm/build/uk_runtime/diagnostics.py | 4 +- .../microcosm/build/uk_runtime/frs_spine.py | 110 +++ .../microcosm/build/uk_runtime/frs_take_up.py | 7 +- .../build/uk_runtime/gate_phase_codes.py | 19 +- .../build/uk_runtime/geographic_clones.py | 67 +- .../build/uk_runtime/geographic_expand.py | 122 ++- .../build/uk_runtime/geographic_identity.py | 98 +- .../src/microcosm/build/uk_runtime/graph.py | 123 ++- .../build/uk_runtime/graph_kernels.py | 62 +- .../build/uk_runtime/graph_national.py | 7 +- .../build/uk_runtime/graph_population.py | 19 +- .../build/uk_runtime/household_lineage.py | 14 +- .../build/uk_runtime/national_frame.py | 8 + .../uk_runtime/private_school_attendance.py | 6 +- .../build/uk_runtime/receipt_counts.py | 16 +- .../build/uk_runtime/rowwise_dataset.py | 53 +- .../build/uk_runtime/salary_sacrifice.py | 21 +- .../build/uk_runtime/spi_band_donors.py | 25 +- .../microcosm/build/uk_runtime/spi_spine.py | 8 +- .../microcosm/build/uk_runtime/spi_support.py | 44 +- .../microcosm/build/uk_runtime/spine_build.py | 85 +- .../tests/engine/uk/test_uk_graph.py | 8 +- .../uk/test_uk_release_input_coverage.py | 1 + .../engine_free/shared/test_country_spec.py | 6 +- .../engine_free/uk/test_uk_cgt_support.py | 7 +- .../uk/test_uk_clone_first_nesting.py | 212 +++- .../engine_free/uk/test_uk_copy_id_table.py | 59 +- .../tests/engine_free/uk/test_uk_frs_spine.py | 361 ++++++- .../uk/test_uk_full_population_graph.py | 85 ++ .../uk/test_uk_full_target_graph.py | 27 + .../uk/test_uk_geographic_expand.py | 420 +++++++- .../uk/test_uk_geographic_identity.py | 151 ++- .../tests/engine_free/uk/test_uk_graph.py | 349 ++++++- .../engine_free/uk/test_uk_graph_evidence.py | 13 +- .../uk/test_uk_high_value_home_support.py | 7 +- .../uk/test_uk_identity_stability_receipts.py | 20 + .../uk/test_uk_rules_engine_provenance.py | 48 +- .../engine_free/uk/test_uk_source_stages.py | 31 +- .../engine_free/uk/test_uk_spi_band_donors.py | 14 +- .../tests/engine_free/uk/test_uk_spi_spine.py | 56 +- .../engine_free/uk/test_uk_spi_support.py | 106 +- .../uk/test_uk_spine_acceptance_receipt.py | 19 +- .../engine_free/uk/test_uk_terminal_gates.py | 65 +- .../uk/test_uk_uc_capital_coherence.py | 3 + .../fixtures/parity/uk_spine/PRODUCED_BY.txt | 2 +- .../parity/uk_spine/sources/fixture.json | 138 ++- .../uk_spine/sources/frs_raw/househol.tab | 272 +++--- .../parity/uk_spine/sources/household.csv | 272 +++--- .../parity/uk_spine/sources/schema.json | 2 + .../fixtures/parity/uk_spine/uk_spine.json | 2 +- .../microcosm_build/uk_clone_first_frames.py | 28 +- ...uild_uk_release_input_coverage_manifest.py | 10 + tools/graph_uk_spine_fixture.py | 74 +- tools/verify_uk_identity_stability.py | 13 +- 76 files changed, 4134 insertions(+), 1418 deletions(-) create mode 100644 changelog.d/931-uk-spine-geographic-expand.changed.md diff --git a/changelog.d/931-band-donors-per-clone.changed.md b/changelog.d/931-band-donors-per-clone.changed.md index 799b10c20b..31777f4114 100644 --- a/changelog.d/931-band-donors-per-clone.changed.md +++ b/changelog.d/931-band-donors-per-clone.changed.md @@ -1,4 +1,4 @@ -The SPI income band donors (`spi_income_band_donors`) are seated within each geographic clone (#931). Where the households carry `household_clone_index`, the stage runs its Efraimidis-Spirakis walk once in each clone, over that clone's candidates, with the same person-keyed uniforms, keys and tie order, and the same seats in every clone (María's rulings of 7 Nov). The seats and the household share's cap read the eligible households counted in distinct `source_household_id` values, so they do not depend on the clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, seats exactly the one-clone donors, carriers and bands. Each donor weighs its band's published taxpayers over the clone count times the band's donor count. A spine without the column, as every spine is today, is one clone. +The SPI income band donors (`spi_income_band_donors`) are seated within each geographic clone (#931). Where the households carry `household_clone_index`, the stage runs its Efraimidis-Spirakis walk once in each clone, over that clone's candidates, with the same person-keyed uniforms, keys and tie order, and the same seats in every clone (María's rulings of 7 Nov). The seats and the household share's cap read the eligible households counted in distinct `source_household_id` values, so they do not depend on the clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, seats exactly the one-clone donors, carriers and bands. Each donor weighs its band's published taxpayers over the clone count times one clone's band donor count. A spine without the column is one clone; the spine's `geographic_expand` stage writes it, 0 on every household of a one-clone spine. - **Funding.** A donor's funding stratum is its region within its clone, a derived stratum (the manifest's `funding_strata` stays `[region]`), so every clone's regions conserve their own mass. One mass scale, the smallest any clone's region allows, serves every clone, so each band keeps one donor weight and the stage gate's and the identity tool's band-mass reconciliations hold as before. The exact-total correction still lands on one incumbent row, never on a donor, so a band's donors stay equal bit for bit at any clone count. The seating nests exactly at any clone count. Clone 0's weights and funding factors nest, its weights as the one-clone weights over the clone count, only while the common scale is the one a one-clone spine takes: one at both, or bound by a region of clone 0. Where another clone's region binds, every clone's donors, clone 0's included, take the lower scale. Even while they nest, the weights nest to rounding, not bitwise: the corrected row, which can sit in any clone, moves by a few ulps of the total, and the 1/K split rounds when the clone count is not a power of two, so uncorrected rows also miss by about one ulp then. The mass record's reason is unchanged, so neither the frame's content identity nor the coverage manifest's restated reason moves. - **Receipt.** The evidence gains `clone_count`, and `funding_strata` lists `household_clone_index` where the column is present (the funding rows' labels then read `region|clone`). `expected_donor_households`, `expected_donor_count` and the donor counts cover every clone; `donor_weight` and `planned_donor_weight` are one clone's weights. Under the receipts' floor of three (`receipt_counts`), the eligible household counts, in all and per band, are counted in source households and rounded in place of the exact counts, each band's being clone 0's, which a one-clone spine counts alike; `seating_scale` reads `full` or `capped`, since an exact scale below one gives back the eligible count; a band's `carrier_mean_age` reads `<3` when its carriers come from fewer than three source households or the band seats fewer than three households per clone; and `carrier_region_counts` and the funding rows' `donor_households` are rounded. The band rows' donor and carrier counts stay exact row counts, which the gate reads. diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index 25693cfae8..a182eedc5e 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,6 +1,6 @@ -The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, or 1 where the column is absent, as on every national spine today. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. +The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, which the spine's `geographic_expand` stage writes, or 1 where the column is absent. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. - **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: for a weight a few ulps above a multiple of 60 (at K > 1, by more than the snap absorbs), or where an exact-total correction or the #970 anchor's and the residential arms' roundings moved a member. So neither the count check nor the family test recounts the fold. At any K, a family of more than three members is held exactly where at least four of its members show their two clone halves at equal weights and every member that does shares one pair sum c, whatever the others carry: its stored count n must be the count of a weight w with fl(w / n) = c. A member that took the split's own fallback correction keeps equal halves at another weight, since the incidence clone halves it exactly, so its family keeps the bracket. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so one of those four or more is uncorrected, and equal halves mean the anchor and the arms left its pair sum at exactly w / n, so c is exactly w / n. The stage counts n on each clone's own weight and gives every member fl(w / n), and the snapped count still rises with w, so this holds at K > 1 as at K = 1. Where every member shares c but fewer than four show equal halves, the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member where the floats either side of c are as wide, and two floats where c sits within two floats of a power of two, below which they are half as wide; c lies in (45 / K, 60 / K] to within the snap, which can leave it a few floats above 60 / K (at most 16 for K up to 15, by design arithmetic over weights at and up to 40 floats above each multiple m × 60 / K, m from 4 to 3,999), and that range holds 16 at K = 3, 8 at K = 6 and 7, and 4 at K = 12 to 15. Every other family keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. The family test, which decides whether a household is a family at all, is held the same way at the K-aware count: a household the rule selects is possibly a family where its count at the folded weight times 1 + 1e-9 exceeds one and possibly whole where its count at the folded weight times 1 − 1e-9 is one, its stored state accepted in between, and the hold's counts decide where it applies. That bracket opens for a household whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies, a whole household or a family of two among them. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a household the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to a few floats next to the count's step, for some counts n. At K = 1 the step sits a float above the multiple, which leaves at most the multiple itself where the family is held exactly, or the multiple and the float either side of it where the hold reaches one float. At K from 2 to 15 the snap puts the step 3 to 17 floats above the multiple of 60 / K, which leaves at most the float on either side of the step where the family is held exactly, or up to three floats from it in the one-float tier, so a family on the multiple itself split into one copy more is reported. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. -- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception and no more, since it makes no one-copy check. Both stages' notes say where the correction lands, and the CGT stage's add that the shared fallback does not make the one-copy check, and that the identity tool's selection recompute can then find a family the stage never made, or miss one it made. The tool reports a family it misses. One it finds that the stage never made, a one-copy household the fallback lifted across 60 / K, moves only `copies_recomputed`, since the bracketed family test cannot tell it from a whole household (see Receipt and gate). The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. +- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception and no more, since it makes no one-copy check. Both stages' notes say where the correction lands, and the CGT stage's add that the shared fallback does not make the one-copy check, so it can put the correction on a one-copy household the check passed over, and that the identity tool's selection recompute can then count that household as a family the stage never made, which the bracketed family test does not report, or miss one it made, which the tool reports. Such a household, lifted across 60 / K, moves only `copies_recomputed`, since the bracketed family test cannot tell it from a whole household (see Receipt and gate). The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. - **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy can carry it only where its corrected weight keeps its one copy and the walk's selection, the one-copy check `copy_rule` names, kept as a reading of the whole-households-first rule, so wherever one of the 64 carries it, the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the rule rerun on the stage's output would have counted a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy, then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. The fallback does not consult that check, so its row can also be a whole household, among them a one-copy household the check passed over; the rerun can then divide a different set. That can happen on synthetic frames, as it could under the rule before, with the weights the rule before gave: where the one-copy household sits one to three ulps below its band's support mass, the identity tool then reports a missing family, and where it weighs 60 at K = 1 and outweighs every copy, the rule rerun counts a family the stage never made, which the tool's family test, held to its bracket, does not report, since the corrected weight lies within a relative 1e-9 of 60. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate unless a selected household left at one copy outweighs it, or ties it from an earlier row, which needs every unselected household to weigh at most 60 / K (to within the K-ulp snap at K > 1). The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. A selected household left at one copy that can carry it also costs a rerun of the selection walk, about 0.41 s at that size, so 64 that the check passes over would take about 26 s; a national frame does not reach that, since its 64 heaviest whole households are unselected ones, far heavier than 60 / K. - **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot and a later one of the 64 can, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them; where none of the 64 can, nothing moves. At stage 42, and at K = 1 too, a selected household left at one copy that outweighs the heaviest unselected household, or ties it from an earlier row, comes first, which needs every unselected household to be as light; where it can carry the correction and keep the split, it takes it even where the heaviest unselected household could have, so those two rows' weights move by rounding amounts. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the hold, the count bracket and the bracketed family test. Against the exact recount it replaces, it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count, nor a family missing, or extra, where a household's re-summed fold lands across the one-copy boundary (60 / K) from its stored state: on the synthetic frames, the one false missing family the full-chain probes found, a two-copy family a float above 60 that folds onto 60, and one false extra family at K = 2. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, where at least four of its members show equal halves and every member that does shares one weight, whatever the others carry: at K = 1 for all but 31 of the 9,995 counts n from 4 to 9,998, and at K from 2 to 15 for every count. Where fewer show equal halves but every member shares one weight, the hold reaches one float, which at K = 1 takes in the multiple itself for 6,393 of those counts, so such a family goes unreported there, as the full-chain test's 900 family does at one clone; at K > 1 it is reported. It no longer reports a family of three or fewer members, or one whose members fold to different weights without four equal halves among them, nor a household split into two copies at 60 / K, which the family test's bracket cannot tell from a whole one. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`, and node keys chain on their inputs' keys, so every spine node from stage 26 on, and every node downstream of the spine, re-keys with them. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. diff --git a/changelog.d/931-isa-tied-balance-rank.fixed.md b/changelog.d/931-isa-tied-balance-rank.fixed.md index 3818fd0efb..f91be7025a 100644 --- a/changelog.d/931-isa-tied-balance-rank.fixed.md +++ b/changelog.d/931-isa-tied-balance-rank.fixed.md @@ -1,7 +1,7 @@ The UK ISA subscriptions stage (`isa_subscriptions`, stage 53) gives every holder tied on a balance one weighted mid-rank (#931; María's ruling 7 of 7 Nov). Half of each subscriber's ISA amount key, and of each Lifetime ISA payer's payment key, is the person's balance rank, blended with a keyed uniform. The rank sorted balances stably and gave each row its own mid-rank, so tied balances ranked by row position, although its docstring said ties share a rank. Ties are common: the copies and clone halves made after the WAS wealth draw carry their root's ISA totals, so a single-adult one ties its root, and QRF draws repeat a few atoms. A root sits before its copies in the table, so it ranked below them, as clone 0 would below its clones once the spine clones households at its start. The stage was also not invariant to row order where balances tie: on a synthetic walk of 400 holders with balances on five atoms, permuting the rows moved 372 amounts, and none with distinct balances. - **The rank.** `_weighted_rank(values, weights, ids)` orders the rows by (value, id). Every member of a tie group takes the group's mid-rank, `(F(x-) + F(x)) / 2` over the total weight. A value held by one row keeps its own, `(c - w / 2) / total`, computed as before. Both callers pass the person ids, so the keyed uniform alone orders tied holders, exchangeably, and each rank is the same bit for bit whatever the row order. -- **No seed, no declaration.** The rank draws nothing, and no declared text says anything about ties, so no seed, salt, manifest key or generated file changes. The band walk and the Lifetime ISA strata stay pooled. +- **No seed; the notes say it.** The rank draws nothing, so no seed, salt or operation key changes. Stage 53's notes now say that tied balances share one weighted mid-rank, so the keyed uniform alone orders them, and that the Lifetime ISA payments, whose operation declares no rank weight, reuse the amounts operation's `balance_rank_weight` (María's ruling of 10 Nov: in the notes, not a declared field). The notes change in `sources.yaml`, `source_stages.json` and the H2 fixture's declarations; notes enter no stage contract, so the stage's contract hash does not move. The band walk and the Lifetime ISA strata stay pooled. - **What moves.** Nothing where no balance ties: on three synthetic frames of distinct balances every output is unchanged, bit for bit. Elsewhere: - Tied holders' ranks move. A singleton's rank is unchanged bit for bit wherever the rows arrive in id order, as the spine's tables do, but its amount can still move, since the band walk and the within-band order are pooled over the holders a tie moves past. - On three synthetic frames with balances on five atoms, every tied holder's rank moved; 248 to 260 of the 363 to 376 tied subscribers and 329 to 344 of the 420 to 430 other subscribers moved amount; and 46 to 56 of the 52 to 68 Lifetime ISA payers, all of them tied, moved payment. No balance moved. diff --git a/changelog.d/931-keyed-tie-breaks.changed.md b/changelog.d/931-keyed-tie-breaks.changed.md index 43060ddd00..aa5262a866 100644 --- a/changelog.d/931-keyed-tie-breaks.changed.md +++ b/changelog.d/931-keyed-tie-breaks.changed.md @@ -7,7 +7,7 @@ Six walks on the UK spine that ordered exactly tied rows by id now order them by - **Stage 44** (`hmrc_cgt_gains_spine`): one uniform per person under the stage's seed (552), salted `hmrc_cgt_gains_spine:rank_tie:`, drawn once per stage call and read by every cell walk, pooled walk, plan and the sub-AEA remainder, which take it as `_ranked`'s keyword-only `tie` (without it, person_id alone breaks ties, as before). The plans and the remainder walk the rank ascending, so a tie's member with the higher uniform takes the lower stratum. - **Stage 47** (`hmrc_cgt_asset_type_spine`): the BADR walk and the main asset type walk, under the BADR seed (555), salted `assign_badr_qualifying_gains:tie:` and `assign_main_asset_type:tie:`. - **Declarations.** The two support splits declare their tie-break as new `seed` and `salt` keys and the receipts walk its `tie_salt`, in the code dictionaries, the strict schema (three operations), `sources.yaml` and `source_stages.json`; the selection, stratum rule, order, ordering, mapping, realization and values texts, the stage notes and the receipts walk's and asset types' receipt order texts say the above, and "no draw, no seed, no salt" and "deterministic, no seed" are gone. The build sidecar's declared seeds register the splits under their salts and the receipts walk's seed under its tie salt. -- **Stage 42's rule B.** The split's walk, `keeps_the_split`'s rerun of it and the identity tool's recompute all pass the declared pair (`cgt_support_tie_break()`); the selection takes no default, so neither the rerun nor the tool can walk ties in another order. The exact-total correction's candidate order (the heaviest households left whole first, tied weights in row order) is unchanged: it picks the row of a rounding-sized correction rather than drawing anything. `copy_rule` names the cap of 64 candidates but not their order; tied weights take row order, so of a whole household's geographic clones, which weigh the same, the one in the earliest row is tried first. +- **Stage 42's rule B.** The split's walk, `keeps_the_split`'s rerun of it and the identity tool's recompute all pass the declared pair (`cgt_support_tie_break()`); the selection takes no default, so neither the rerun nor the tool can walk ties in another order. The exact-total correction's candidate order (the heaviest households left whole first, tied weights in row order) is unchanged: it picks the row of a rounding-sized correction rather than drawing anything. `copy_rule` names the cap of 64 candidates and their order, heaviest first, tied weights in row order, so of a whole household's geographic clones, which weigh the same, the one in the earliest row is tried first. - **No nesting promised.** All six walks are pooled. The uniform is a function of each row's own id, so clone 0's equals the `K = 1` row's and every clone draws its own. - **What moves.** Nothing where no exact tie touches a cut or a position schedule. Elsewhere, counted on synthetic frames: - At stage 42, a band's crossing household when it sits in a tie group: with three clones per household, 2 to 8 of about 850 households per frame changed copy count, and clone 0, which carried the most families in all six frames, carries the most in one. At one clone none moved on those frames, whose synthetic tied families are light, most members at or under 60, the weight above which a selected household takes a second copy. Where a band's crossing falls in a tie group whose members weigh more, as a high-value family's can, the keyed order changes which members are copied at one clone too. diff --git a/changelog.d/931-shared-clone-expansion-core.added.md b/changelog.d/931-shared-clone-expansion-core.added.md index efc7a11e99..22cd6e0456 100644 --- a/changelog.d/931-shared-clone-expansion-core.added.md +++ b/changelog.d/931-shared-clone-expansion-core.added.md @@ -25,4 +25,4 @@ A country-neutral clone expansion core, `microcosm.build.clone_expansion` (#931; - **Tests.** - `tests/engine_free/shared/test_clone_expansion.py` runs the core under neutral and US-shaped declarations on invented frames. The US-shaped frame weights households and people, each split with a record of its own, and its clones, labelled from their clone index, pass the shared two-channel invariant `outer_stage_runtime.assert_clone_expansion`. Narrow ids keep their dtype at every K, and each refusal, the dtype ones included, is checked to print no id or weight. - `tests/engine_free/uk/test_uk_clone_expansion_core.py` runs it under the UK's declarations: the copy-id table's `geographic_support` mint and `require_below`, the ladder's clone-index columns, `source_household_id`, and design weights on households. At K = 1, 2, 3, 7 and 15 it reproduces the clone-first stand-in `clone_first_frame`, which is written without the core, bit for bit on the ids, the household clone index, the source household id and the weights. -- **First caller.** The UK's geographic expansion step (`microcosm.build.uk_runtime.geographic_expand`, G1 of the clone-first spine) calls the core with the UK's declaration, `UK_GEOGRAPHIC_CLONES`. Its mint and its `require_below` are named functions in that module that call the copy-id table, so the derived binding rule binds the table to the key of the stage that runs the step. The UK test above now takes that declaration from the step instead of keeping a copy of its own. No stage runs the step yet, so the core moves no output and no node key. +- **First caller.** The UK's geographic expansion step (`microcosm.build.uk_runtime.geographic_expand`, G1 of the clone-first spine) calls the core with the UK's declaration, `UK_GEOGRAPHIC_CLONES`. Its mint and its `require_below` are named functions in that module that call the copy-id table, so the derived binding rule binds the table to the key of the stage that runs the step. The UK test above now takes that declaration from the step instead of keeping a copy of its own. The spine's `geographic_expand` stage runs the step, at one clone until the spine build takes a clone count, and binds the core into its key (`931-uk-spine-geographic-expand`). diff --git a/changelog.d/931-uk-engine-blocks-pool-shares.changed.md b/changelog.d/931-uk-engine-blocks-pool-shares.changed.md index 10539bea41..d31cbd700a 100644 --- a/changelog.d/931-uk-engine-blocks-pool-shares.changed.md +++ b/changelog.d/931-uk-engine-blocks-pool-shares.changed.md @@ -3,4 +3,4 @@ Per-clone engine blocks now stand for the pool when the pool's clones differ (#9 - **What exact means.** `engine_population_representation.exact` now holds only when every block applied its correction, passed its checks and got a clean verdict, and then means equal to one engine over the pool to float32 rounding, not bitwise. `exact_basis` (`pool_share_injection_with_population_guard`) names this basis, and `exact_scope`, which the release's `engine_measurement` limitation copies, is rewritten. A receipt without `exact_basis` predates the change, when exact meant identical copies. The measure step records a violation, which leaves the run not releasable. The release verdict, the local pre-flight, the dense assembler and the data contract read the same fields as before; none requires `exact_basis`. - **Spine batches.** A spine engine call whose frame carries `household_clone_index` with more than one clone runs one simulation per clone, each reading the pool's shares (`c_b = S_b/S`) and judged under a pass that raises. A frame without the column, or with one clone, takes one simulation as before. Stage 52 (`private_school_attendance`) restates policyengine-uk's attendance over the whole frame from household-local engine values, with microdf's `MicroSeries.percentile_rank` and the engine's own parameters. Over one simulation it stores the engine's flags after checking the restatement bit for bit; clone by clone it stores the restatement. Its evidence gains `rank_route` and `engine_parity_checked`. - **The shared pieces.** The country-agnostic machinery (pool shares, the checks, injection, the formula-run recorder, judging, `EngineBlock` and batches) is `microcosm.frame.adapters.engine_population`, and the UK declaration is `microcosm.frame.adapters.policyengine_uk_population`. Each block and batch first checks the declaration against the installed policyengine-uk; a single block and an unbatched spine call never read it. A re-pin that changes the whole-population list or a pinned formula source therefore leaves every dense block uncorrected and the run not releasable, and refuses every batched spine call, until the declaration is reviewed again. -- **What moves.** No spine frame carries `household_clone_index` yet, so every spine call takes one simulation and no spine data moves (frame, H5, stored flags), but stage 52's evidence gains its two fields on both lines, so its stage-evidence digest moves. Identical copies take `c_b` within float64 rounding of 1, so the correction leaves their measures bitwise as they were, and a single block, the national line's, gains none of the new receipt fields. Today's dense receipts gain `exact_basis`, `share_correction`, `population_reach` and `annual_income_projection_factors`; `block_sensitivity` now names engine variables, so `present_in_this_run` can list them, drops the SLC entry and gains `corrected_formulas`; and each block's resolver receipt gains `engine_pool_shares` and `engine_population_reach`. Every spine stage and the full and national target kernels re-key, and the engine-calling stages also bind the two new modules. The annual-income factor, the one data change, has its own fragment. +- **What moves.** Every spine is one clone (`geographic_expand` writes `household_clone_index`, 0 throughout), so every spine call takes one simulation and no spine data moves (frame, H5, stored flags), but stage 52's evidence gains its two fields on both lines, so its stage-evidence digest moves. Identical copies take `c_b` within float64 rounding of 1, so the correction leaves their measures bitwise as they were, and a single block, the national line's, gains none of the new receipt fields. Today's dense receipts gain `exact_basis`, `share_correction`, `population_reach` and `annual_income_projection_factors`; `block_sensitivity` now names engine variables, so `present_in_this_run` can list them, drops the SLC entry and gains `corrected_formulas`; and each block's resolver receipt gains `engine_pool_shares` and `engine_population_reach`. Every spine stage and the full and national target kernels re-key, and the engine-calling stages also bind the two new modules. The annual-income factor, the one data change, has its own fragment. diff --git a/changelog.d/931-uk-fits-fold-geographic-clones.added.md b/changelog.d/931-uk-fits-fold-geographic-clones.added.md index 88db8e49e7..9883d358aa 100644 --- a/changelog.d/931-uk-fits-fold-geographic-clones.added.md +++ b/changelog.d/931-uk-fits-fold-geographic-clones.added.md @@ -1,7 +1,7 @@ The UK fits trained on the frame's FRS rows now fold geographic clones into each survey record before they fit (#931). This covers the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. - **How the fold works.** Each fit passes what it reads: its predictors and targets, or the columns they derive from. A record's clones that agree on it keep one training row, the lowest-id one among them (clone 0 whenever clone 0 is selected and agrees, whatever the row order), with the weight of those clones summed exactly. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own, each variant with its own clones' summed weight. SPI, high-value and CGT copies stay rows of their own, as now. -- **No change today.** Today's spine has no geographic clones, so every fit's rows and weights stay as they were, value for value. +- **No change today.** Today's spine is one geographic clone (`geographic_expand` runs at K = 1), so every fit's rows and weights stay as they were, value for value. - **What changes once the spine clones before enrichment:** - the fits see each record once wherever its clones agree, and every clone's value at its share wherever they differ; - the reference transport meets no duplicate twins from agreeing clones; diff --git a/changelog.d/931-uk-floors-count-true-sources.changed.md b/changelog.d/931-uk-floors-count-true-sources.changed.md index 2617489db2..115854ede3 100644 --- a/changelog.d/931-uk-floors-count-true-sources.changed.md +++ b/changelog.d/931-uk-floors-count-true-sources.changed.md @@ -1 +1 @@ -Every UK floor now counts true source households, the survey household each copy and clone keeps in `source_household_id`, through one reader (#931). The reader is the new country-neutral `microcosm.build.source_lineage.record_sources`: a country declares its copy lineage once (the entity copies are made of, its source column and each copying step's marker and ordinal columns), and every record of any entity reads its source through household membership. `receipt_counts.record_source_households` is the UK's entry on `household_lineage.UK_COPY_LINEAGE`, and `receipt_counts.SOURCE_HOUSEHOLD_COLUMN` is the shared `source_household_id`, no longer `household_source_id`, the support channel's immediate parent. The reader refuses a frame without the source column, and a null source; a stage that runs before the SPI support channel says so (`before_copies`), and its households are then their own sources unless the frame already carries copy lineage. The take-up signal receipts, the apprentice receipt and the annual income receipts (stages 10 to 12 and 15), the SPI pay floor and pension coherence (which read `household_source_id` without the null refusal), company cars, the SPI benefit coherence, UC take-up's pools, the legacy closure, salary sacrifice's receipt and bonus donor floor, Carer's Allowance, the trading allowance, the support floor, the single-year age report, the dataset-size evaluation, the sampled-rung private-rent region floor, and the council-tax, BRMA-distribution, weighted-integrity, benefit-cap and UC-relationship tools count through it. The two columns agree on every household row of today's spine and pool, so no spine output moves. The single-year age report and the dataset-size evaluation now count a zero-weight row as carrying nothing and a household's copies once, so their support counts can fall; the weighted-integrity tool's carriers are now distinct sources, not rows. The five finite stages that declared `household_source_id` declare `source_household_id`, UC take-up's pooling text and the Carer's Allowance and support-floor notes name it, and a static test refuses any distinct count of `household_id`, `person_household_id` or `household_source_id` in the UK runtime and tools outside two named structural checks. The receipts' counting modules (`receipt_counts`, `household_lineage`, `source_lineage`) are bound into the implementation hash of every stage whose module or helpers import one of them; eight stages that counted through `receipt_counts` did not bind it, so an edit to it alone left their cache keys in place. +Every UK floor now counts true source households, the survey household each copy and clone keeps in `source_household_id`, through one reader (#931). The reader is the new country-neutral `microcosm.build.source_lineage.record_sources`: a country declares its copy lineage once (the entity copies are made of, its source column and each copying step's marker and ordinal columns), and every record of any entity reads its source through household membership. `receipt_counts.record_source_households` is the UK's entry on `household_lineage.UK_COPY_LINEAGE`, and `receipt_counts.SOURCE_HOUSEHOLD_COLUMN` is the shared `source_household_id`, no longer `household_source_id`, the support channel's immediate parent. The reader refuses a frame without the source column, and a null source; a stage that can run before the first copying step says so (`before_copies`), and on a frame without the source column or copy lineage its households are their own sources; on the clone-first spine that step is `geographic_expand`, which writes the column, so a stage after it that says so reads it. The take-up signal receipts, the apprentice receipt and the annual income receipts (stages 10 to 12 and 15), the SPI pay floor and pension coherence (which read `household_source_id` without the null refusal), company cars, the SPI benefit coherence, UC take-up's pools, the legacy closure, salary sacrifice's receipt and bonus donor floor, Carer's Allowance, the trading allowance, the support floor, the single-year age report, the dataset-size evaluation, the sampled-rung private-rent region floor, and the council-tax, BRMA-distribution, weighted-integrity, benefit-cap and UC-relationship tools count through it. The two columns agree on every household row of today's spine and pool, so no spine output moves. The single-year age report and the dataset-size evaluation now count a zero-weight row as carrying nothing and a household's copies once, so their support counts can fall; the weighted-integrity tool's carriers are now distinct sources, not rows. The five finite stages that declared `household_source_id` declare `source_household_id`, UC take-up's pooling text and the Carer's Allowance and support-floor notes name it, and a static test refuses any distinct count of `household_id`, `person_household_id` or `household_source_id` in the UK runtime and tools outside two named structural checks. The receipts' counting modules (`receipt_counts`, `household_lineage`, `source_lineage`) are bound into the implementation hash of every stage whose module or helpers import one of them; eight stages that counted through `receipt_counts` did not bind it, so an edit to it alone left their cache keys in place. diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md index 2d92e5452e..286614a2ec 100644 --- a/changelog.d/931-uk-helper-bindings.changed.md +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -18,7 +18,7 @@ Every UK spine stage now hashes each first-party module its transform computes t Listed instead, each with a reason a test checks: - the core modules every node runs through, where the rule stops, since binding one would re-key every node on any edit to it. They are the Frame's three modules (`microcosm.frame.bundle`, `schema` and `weights`), which a stage's transform meets all three or none of; the graph executor's (`microcosm.graph.executor` and `population`); and the modules that compute node keys (`keys`, `decl` and `kernel`), which no stage's transform meets. No spine key hashes them. H2 cannot catch an edit to the Frame, since the graph's spine and the legacy stage plan both run through it, and H1, whose fixtures run four wrapped kernels through `run_graph` against stored bytes, catches one only on those kernels' frames; an edit that changes a UK value is otherwise left to the Frame's own tests, and a test pins that `Weights.total`, which the UK exact-total corrections target, is NumPy's sum of the weights. One to the executor's modules that changes a value on H2's fixture fails H2, which compares the graph's spine with the legacy stage plan's. One that changes what `node_key` or `source_hash` returns changes every key. A test checks that the walk from `run_graph` meets each of them. `microcosm.graph.canonical` also encodes every key, but `regional_home_values` draws through it, so it is bound there instead. That stage draws through `microcosm.graph.randomness`, whose contract asks for `SeedSource.KEYED`. Every spine stage kernel, that stage's included, keeps `SeedSource.NONE`, as María ruled on 9 Nov: its stream is a stage parameter, which its contract hash covers, and it reads no generator the executor seeds; - `microcosm.frame.rules`' `assert_rules_engine_country`, the fourth refusal-only check, which every stage that takes an engine calls first; - - `SOURCE_HOUSEHOLD_ID_COLUMN` from `microcosm.frame.concepts`, CopyLineage's default source column, the second listed name. The support channel writes the column by its literal name. Every stage that reads the constant declares the column by that name, which its key hashes, and counts before the copies only beside a copy column, or declares neither, so a renamed constant leaves the lineage readers to refuse or count as before. + - `SOURCE_HOUSEHOLD_ID_COLUMN` from `microcosm.frame.concepts`, CopyLineage's default source column, the second listed name. `geographic_expand` writes the column through the constant and declares it by its literal name, and its transform refuses other outputs, so a renamed constant stops the spine build before any node runs. Every stage after it that reads the constant is given the column, and one that counts before the copies the clone index too; `frs_annual_income_spells`, before it, is given neither and counts each household as its own source (`931-uk-spine-geographic-expand`). Still outside the rule: calls through any other object whose class the code never names, a name looked up by a computed string, and data files read by path. - **The engine.** The 16 stages that call the engine compute through the `PolicyEngineUKEngine` the builds inject, whose class no stage's transform names. As María ruled on 9 Nov, a second walk is rooted at that class, the one `graph_kernels`' hermetic H2 builder and the spine build both construct (checked). It follows the methods a stage's code names on the engine, by attribute on any receiver or through `getattr`, with the constructor and every method they call on `self`, and then the methods its own reach names on any receiver outside the engine's classes, as a helper the engine hands itself to might call, to a fixed point, at the same scope and with the same tables. The stage rule's tests hold what it reaches as part of the stage's walk. It matches methods by name, as the engine-call scan does, so it can only over-count. The engine stages call `materialize`, `variable_metadata` and, through `getattr` in `private_school_attendance`, `parameters` and `simulation_batches`. The walk reaches the three hand-bound engine modules and one more, `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. That module is the new `_ENGINE_COMPUTE_MODULES` group in `graph_kernels`, which every engine stage binds and which a test derives exactly. It sits beside `_ENGINE_MODULES`, which no other stage binds, because the SPI spine's three stages and the band donors bind `frame.materialize` for their own tables; any other stage binds it only where its own transform computes through it (checked). The engine reaches the concept vocabulary (`policyengine_uk_concepts`, `concept_mapping` and `concepts`) and `frame.rules` only through methods no engine stage calls (`concept_mapping` and `export_contract`), so none is bound for the engine. No spine stage hashes the concept vocabulary, and only `uc_take_up` and `pension_credit_take_up` hash `frame.rules`, for the protocol they type their engine by. The hermetic fixture's transform resolver no longer names the engine as held. @@ -58,8 +58,8 @@ Every UK spine stage now hashes each first-party module its transform computes t - the national gates: `gates`, microcosm.calibrate's `artifacts`, `matrix`, `registry`, `target`, `hierarchy`, `monetary_binding` and `solve`, `graph_targets`, `graph_population`, `national_frame`, `etb_services`, `cgt_projection`, `donor_uprating`, `gate_scopes` and the canonical encoder, and everything the gate registry's bindings compute through (see Gate batteries); - the national readback: `artifact_files`, `trace`, `frame.materialize`, the engine adapter, `geography_ladder`, `graph_population`, `hmrc_uprating` and the canonical encoder. - The LAD23 names' pin stays listed, now for every kernel that reads it. The geographic clone index columns' name (`rowwise_dataset.ladder_clone_index_column`) is listed for the calibrated and national gates, whose support-family fold reads it: only the dense expand writes those columns, and it hashes `rowwise_dataset`; the national frame has no ladder, and the fold keys on the column only where the table carries it. It is listed for the dense measures and the national problem too (see Target and problem kernels). `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside implementation hashes: the national gates' and readback's, and for three of them the national problem's. -- **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Where a phase's gates read the engine their kernel hands them, the walk also follows the methods they call on it, from the engine's class, as for an engine stage (see The engine): the spine gates' rules engine is the spine build's `PolicyEngineUKEngine`, and the full gates' engine is the driver's coverage engine. Another test checks each part of that model against the code, the engines among them. So the assembled gate's own key, which the spine stages after it chain on, carries nothing only the transferred phase computes through, and the preflight's, which the dense solve chains on, nothing only the terminal phase does. Key chaining still re-keys them wherever a node upstream hashes the edited module. Of the 17 modules only the transferred phase computes through, stages at or before `frs_brma`, the assembled phase's last, hash `stochastic_assignment` and `take_up_contract`, so an edit to either re-keys the assembled gate; no stage before it hashes the other 15. Of the 38 only the terminal phase computes through, most re-solve too, through the spine stages, the target and problem kernels, the population kernels or the dense solve, which hash them as well. Eight never re-solve, on any line: `high_value_home_gates`, `microcosm.diagnostics`' catalogs and schema, `calibrate.hierarchy_labels`, the target loss attribution, `input_mass`, `cgt_projection` and `source_runtime`, so a disclosure fix between two builds that must reuse the solve still belongs in `high_value_home_gates`. `uk_runtime.diagnostics` re-solves only the national line, whose problem kernel hashes it. `calibration_run` and `release_certification` re-solve only on a bound spine, whose kernel hashes both, and seven modules that only the spine's nodes hash before the solve re-solve only in the in-graph build. The preflight's own code, the shared ledger's parity report, is also hashed by the full build's target compilation, selection, measures and problem, by the national line's targets, problem and gates and, in the in-graph build, by the `etb_services` and `lcfs_consumption` stages and the transferred gate, so an edit to it re-keys those too. + The LAD23 names' pin stays listed, now for every kernel that reads it. The geographic clone index columns' name (`geographic_clones.ladder_clone_index_column`) is listed for the calibrated gates, whose support-family fold reads it on the dense frame: `geographic_expand` writes those columns on the spine and the dense expand only on a spine without them, and both hash `geographic_clones`, as do the national gates, whose fold reads them on the national frame (`931-uk-spine-geographic-expand`). It is listed for the dense measures and the national problem too (see Target and problem kernels). `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside implementation hashes: the national gates' and readback's, and for three of them the national problem's. +- **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Where a phase's gates read the engine their kernel hands them, the walk also follows the methods they call on it, from the engine's class, as for an engine stage (see The engine): the spine gates' rules engine is the spine build's `PolicyEngineUKEngine`, and the full gates' engine is the driver's coverage engine. Another test checks each part of that model against the code, the engines among them. So the assembled gate's own key, which the spine stages after it chain on, carries nothing only the transferred phase computes through, and the preflight's, which the dense solve chains on, nothing only the terminal phase does. Key chaining still re-keys them wherever a node upstream hashes the edited module. Of the 17 modules only the transferred phase computes through, stages at or before `frs_brma`, the assembled phase's last, hash six: `stochastic_assignment` and `take_up_contract`, and, since `frs_hmrc_spine_leaves` and `frs_annual_income_spells` run before it (`931-uk-spine-geographic-expand`), `frs_annual_income`, the vendored-fact reader, the chronicle feed and its schema epochs. So an edit to any of the six re-keys the assembled gate; no stage before it hashes the other 11. Of the 38 only the terminal phase computes through, most re-solve too, through the spine stages, the target and problem kernels, the population kernels or the dense solve, which hash them as well. Eight never re-solve, on any line: `high_value_home_gates`, `microcosm.diagnostics`' catalogs and schema, `calibrate.hierarchy_labels`, the target loss attribution, `input_mass`, `cgt_projection` and `source_runtime`, so a disclosure fix between two builds that must reuse the solve still belongs in `high_value_home_gates`. `uk_runtime.diagnostics` re-solves only the national line, whose problem kernel hashes it. `calibration_run` and `release_certification` re-solve only on a bound spine, whose kernel hashes both, and seven modules that only the spine's nodes hash before the solve re-solve only in the in-graph build. The preflight's own code, the shared ledger's parity report, is also hashed by the full build's target compilation, selection, measures and problem, by the national line's targets, problem and gates and, in the in-graph build, by the `etb_services` and `lcfs_consumption` stages and the transferred gate, so an edit to it re-keys those too. - The spine gates' hash binds the 12 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants`, `calibrate.geography_constants` and the engine adapter, through whose `PolicyEngineUKEngine.enum_domain` each phase's enum gates resolve a domain no artifact supplies, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. The spine gate kernel refuses a stale or missing `_code`, as the full gates do (see The phase codes, below). - The full gates' hash binds the 26 modules both phases compute through. It loses `local_rowwise`, `weighted_integrity` and `diagnostics`, which only the terminal phase computes through. It binds the spine and local gate scopes and the scoped gate manifest, which the preflight computes through, in a new `gate_scopes` module, not in `calibration_run`, the national calibration runner they lived in. They moved there unchanged, and `calibration_run` re-exports them, as microcosm#1492 moved the target kernels' two functions into `ledger_surface`, so in the in-graph build an edit elsewhere in the runner re-keys neither the preflight nor the solve. The bound spine, the national gates and the certification, whose code reaches them too, hash `gate_scopes` as well. The calibrated gates' `terminal_code` binds `calibration_run` itself, for the admin totals and CGT projection artifact their gate context is built with. The preflight's release input coverage gate computes through `stage_evidence`, `parity_reference`, `release_identity` and `release_input_coverage`, which its key used to miss, so on a bound spine an edit to any of them now re-keys the preflight and re-solves; in the in-graph build so does one to `parity_reference` or `release_identity`, while the other two already re-solved through the spine stages that hash them. The preflight node carries a new `preflight_code`: the shared ledger's registry parity, behind the ledger compile parity gates only the preflight declares. The calibrated node's `terminal_code` now hashes 38 modules. `high_value_homes` and `calibrate.geography_constants` move into the shared hash, since the preflight computes through them too; the target kernels hash both, so an edit to either already re-solved. The kernel refuses a stale or missing `preflight_code`, as it refuses a stale `terminal_code`. - The phase codes. As María ruled on 9 Nov (late), the four functions that compute them and their two tables live in a small module of their own, `uk_runtime.gate_phase_codes`, unchanged in what they hash, with one stale-code check, `require_current_phase_code`, which both batteries' kernels call before they read anything of their node but its parameters. In `graph_evidence` and `graph_terminal`, which define the gate kernels and which every spine stage kernel, the dense population kernels and the export kernels hash too, an edit to a phase's list of modules, which the battery test demands whenever a phase's evaluators gain one, re-keyed every node whose kernel hashes either file, and by key chaining every node after one: for a spine phase's list every node of the spine and of the in-graph build and all but the checkpoint of the local line on a bound spine, for a full phase's list the preflight, the dense solve and every node after them. Now only the graph construction, which sets each node's `_code`, and the check read the module, and no kernel hashes it: the check is listed refusal-only, since it reads its parameters and the phase's code and refuses, writing nothing, and a phase's own code reaches its key through the parameter, which `node_key` hashes. So an edit to a phase's list moves only that phase's parameter, which re-keys that phase's node and what follows it. `source_hash` leaves the kernels' listed core names: only the full gates' run named it, through the phase-code table, which only the check reads now. The spine gates' run now refuses a stale or missing `_code` too, a new refusal, so no gate evaluates its phase under a key that names other phase code. A record stored under a gate node's key is still reused without running its kernel, so the check does not guard a cache hit. Tests check that no UK kernel's key covers the module; that both gate kernels call the check first, on their own battery's table, with no return before it and their phase bound once, before it; that each spine phase refuses a stale and a missing code and passes its node's own; and that the check refuses a code on a phase its battery's table leaves out, and a battery with no table. diff --git a/changelog.d/931-uk-spine-geographic-expand.changed.md b/changelog.d/931-uk-spine-geographic-expand.changed.md new file mode 100644 index 0000000000..57a8e50499 --- /dev/null +++ b/changelog.d/931-uk-spine-geographic-expand.changed.md @@ -0,0 +1,35 @@ +The UK spine gets its geographic clones after its source stages (#931). `frs_hmrc_spine_leaves` and `frs_annual_income_spells`, which join the raw FRS tabs on raw person ids, move ahead of `frs_take_up`, and two new stages follow them: `geographic_expand` (G1) gives every FRS household K geographic clones, and `geographic_identity` (G1b) keys each clone. `frs_take_up`, `frs_person_draws`, `frs_household_draws` and `frs_brma` then run on every clone (María's ruling of 10 Nov, decision 3). The spine build runs one clone, so no value moves. Positional stage numbers in the #931 fragments, comments and tests still follow the order before this change: 14 is `frs_hmrc_spine_leaves`, 15 `frs_annual_income_spells`, 10 to 13 the draws and `frs_brma`, and 16 onward as before. + +- **Order.** The two moved stages read only employment and self-employment income, age and gender. The draws read none of their outputs and write none of their inputs, and none of these stages sets weights. `frs_brma` stays the assembled checkpoint's last stage, so the assembled phase now covers the moved stages, G1, G1b and the draws, and the assembled gate admits `spi_support_channel`. +- **Declarations.** The sources schema and `ALLOWED_SOURCE_OPERATION_KINDS` gain two strict operation kinds. + - `expand_geographic_clones` declares the entity, the clone-index columns, the ordinal bound of 100, the copy-id table's `geographic_support` field at radix 10^9, clone-major order, an equal split of design weights, the household source columns, conservation to the split's rounding, a declared factor of 1.0 and a mass reason that names no clone count. + - `key_households_by_clone_path` declares the source `frs` and `frs_release.json`, which G1b also declares as its one artifact with `runtime_sha256_required`, so the release's bytes enter its stage contract. + + Each transform refuses any other declaration of its operation. G1 refuses any artifact, and G1b any but that one. +- **Clone count.** No declaration names K. `uk_spine_graph(geographic_clones=K)` puts it on G1's node, a whole number from 1 to 100, and refuses K above 1 for a manifest without G1. So K enters that node's key and every later one, never a stage contract. + - G1 is an EXPAND that conserves mass at design weights. Its kernel refuses a transform whose clone count differs from its node's, printing only the two counts. + - The spine build runs G1 at K = 1 and builds its engine with `clones=1`. `_rules_engine`'s `clones` defaults to None, so the full build's engine installs are unchanged. + - The H2 registry builds G1 and its engine from the fixture's `config.geographic_clones`. + - The dense line's `uk.full.identity` declares the geography key as a rewrite where the spine carries it, so the dense graph still compiles. +- **Source columns.** `source_household_id`, `source_year` and `source_household_key` are G1's outputs. `spi_support_channel` no longer declares or writes them, and its copies inherit them. It and the dense expansion default them from a household's own ids only on a table without `household_clone_index`, and refuse, naming the columns, a clone-indexed table that lacks any of them. `source_person_id` and `source_benunit_id` are not minted (decision 5). Their names live in the `geographic_clones` leaf beside the clone-index names (`UK_GEOGRAPHIC_SOURCE_COLUMNS`, `SOURCE_YEAR_COLUMN`, `SOURCE_HOUSEHOLD_KEY_COLUMN`), and so does the geography key's (`GEOGRAPHY_HOUSEHOLD_KEY_COLUMN`, which `atomic_area_support.IDENTITY_COLUMN` re-exports). So G1, G1b, the SPI support channel and the dense line read one definition, which their keys hash. +- **Counting and folding.** `frs_take_up`, `frs_person_draws` and `frs_household_draws` are given `source_household_id` and `household_clone_index`, so their receipts count true source households on every clone. `spi_employment_shell`, `spi_housing_shell`, `uc_capital_coherence` and `property_components` are given `household_clone_index`. +- **Root carriers.** `frs_spine` writes two typed household carriers, and `council_tax_band` itself is unchanged: + - `imd_decile` (int64): the household's decile on its own nation's index of multiple deprivation (IMDE, IMDW, IMDS or IMDN), 1 to 10, else 0; + - `council_tax_band_observed` (bool): False in Northern Ireland, for a code outside 1 to 9, for a band the household's region does not use and for an unknown region. + + The root refuses a household tab without one of the four IMD variables, naming it. Both carriers leave at the release boundary and stay out of every receipt and of the spine sidecar's nonzero shares. The root binds `calibrate.geography_constants`, from which both read the household's nation. The area assignment of #931 group (c) is their first reader. +- **microdf.** microdf-python's version joins every spine node's numerical dependencies (`not-installed` when absent), and the sidecar records it as `microdf_python` beside `engine_batches` when a call batches (decision 2). `pyproject.toml` and `uv.lock` are unchanged; the next lock bump declares it. +- **Texts.** + - The `copy_rule` of both support splits, `high_value_home_support` (stage 26) and `cgt_support_split` (stage 42), names the order the exact-total candidates are tried in. + - The notes of `isa_subscriptions` (stage 53) say tied balances share one weighted mid-rank and the Lifetime ISA reuses `balance_rank_weight` (decision 1), and `docs/uk-isa-subscriptions-1415.md` steps 3, 5 and 6 follow the code. + - The weight rule of `spi_income_band_donors` (stage 17) names one clone's donor count, and its draw a spine without the clone index. + - The bonus draw of `salary_sacrifice` (stage 48) draws within the adult's own clone from cells pooled over every clone, and says the lowest cell joins the one above. + - The source and notes of `spi_support_channel` (stage 16) and `private_school_attendance` (stage 52), the notes of `spi_income_band_donors`, `cgt_support_split`, `salary_sacrifice` and `uc_capital_coherence`, and `frs_spine`'s notes (the two carriers) say what each does on a clone-first spine. + - `childcare_copy_take_up`'s notes say `frs_take_up` draws each geographic clone on its own ids before any other copy exists, and that geographic clones keep those draws. + - Earlier #931 fragments that said G1 was unwired, or that no spine carries a clone index, now say what this change does. +- **Bindings.** The national gates hash `geographic_clones`, since their support-family fold reads the clone index on the national frame of a clone-first spine. The copy-id readers gain G1. The stage table keeps only the source column's name, with its reason rewritten, and the kernel table's clone-index reason no longer covers the national gates. +- **Dense and national.** The dense expansion refuses a spine carrying G1's clone index at any `n_clones`, naming #931 group (c), which builds the dense line on one. No national or dense build runs on such a spine before (c) (decision 4). The coverage manifest's new `geographic_expand` family requires G1's mass record under its declared reason, so a release from a spine built before this change fails the coverage gate too. +- **What moves.** No value at one clone. On the synthetic H2 fixture, the legacy oracle's final frame keeps the parent's rows, existing columns, weights, strata and mass records, in a new column order. It gains the three clone indices at 0, `geography_household_key` (which copies made after G1b repeat until (c)), the two root carriers and G1's mass record, and the funding strata of `spi_income_band_donors` read `REGION|0`. + - Every UK spine node re-keys. Seven stage contracts move with their declarations (`frs_spine`, `spi_support_channel`, `spi_income_band_donors`, `high_value_home_support`, `cgt_support_split`, `salary_sacrifice`, `private_school_attendance`), and G1's and G1b's are new. + - Outside the UK, adding the two kinds to `source_manifest` re-keys the shared transport gate battery, so NZ's terminal gate and its export and package nodes re-key. The US, NZ, BE and AM specs keep their fingerprints. + - `source_stages.json`, the H2 fixture and the coverage manifest are regenerated. diff --git a/changelog.d/931-uk-stages-ready-for-clones.changed.md b/changelog.d/931-uk-stages-ready-for-clones.changed.md index 11c6b02338..856867673b 100644 --- a/changelog.d/931-uk-stages-ready-for-clones.changed.md +++ b/changelog.d/931-uk-stages-ready-for-clones.changed.md @@ -1,11 +1,11 @@ -The UK spine gains G1 and G1b, the steps that will give it geographic clones, without wiring them yet, and three of its stages are made ready for clones: salary sacrifice draws SPI bonuses within each clone, UC capital counts its donors in survey records, and the CGT support split reads the clone count from a module of its own (#931). No output moves: every spine is one clone today. +The UK spine gains G1 and G1b, the steps that give it geographic clones (`931-uk-spine-geographic-expand` declares and wires them), and three of its stages are made ready for clones: salary sacrifice draws SPI bonuses within each clone, UC capital counts its donors in survey records, and the CGT support split reads the clone count from a module of its own (#931). No output moves: every spine is one clone today. - **Clone-index names.** `ladder_clone_index_column`, `ARTIFACT_CLONE_INDEX_COLUMN` and `geographic_clone_count` move unchanged to `microcosm.build.uk_runtime.geographic_clones`, which imports nothing first-party. `rowwise_dataset` and `cgt_support` re-export them, so every importer keeps working. The module joins the spine's counting modules, so every stage that counts by lineage hashes it. The CGT support split, which reads the clone count, binds it as a helper. The full build's population and export kernels, which read the names, hash it too: `uk.full.expand` and the other population kernels, `uk.full.identity`, `uk.full.property.value`, `uk.full.private_rent` and both export descriptions. - **Geographic expansion (G1).** `geographic_expand.expand_uk_geographic_clones(frame, clones=K)` and its stage transform, `UKGeographicExpandStageTransform(stage, clones)`, turn a spine frame into K clones of every FRS household, clone-major. They call the shared clone expansion core, as María ruled on 10 Nov, with the UK's declaration, `UK_GEOGRAPHIC_CLONES`: - clone `k` takes `id + k × 10^9` on all five id columns, in the copy-id table's `geographic_support` field, through two named functions that call the table (`mint_geographic_clone_ids`, `require_room_for_geographic_clones`); clone 0 keeps the frame's ids; - each table carries its clone index (`person_clone_index`, `benunit_clone_index`, `household_clone_index`), 0 to K - 1, with exactly K values; - each clone takes `w / K` of its household's design weight, the same float in every clone, and the mass log gains one record whose reason names no clone count; - - each household carries the SPI support channel's source columns: `source_year` (the frame's time period) and `source_household_key` (`"{source_year}:{source_household_id}"`, the values that channel writes, as Python strings in an object column, the dtype the legacy plan gives it; the graph casts it to its string dtype), written before the clones are made, and `source_household_id`, its clone-0 id, which the core writes. + - each household carries the source columns the SPI support channel wrote before G1 took them over: `source_year` (the frame's time period) and `source_household_key` (`"{source_year}:{source_household_id}"`, the values that channel wrote, as Python strings in an object column, the dtype the legacy plan gives it; the graph casts it to its string dtype), written before the clones are made, and `source_household_id`, its clone-0 id, which the core writes. It refuses, by name or by count only: - a clone count outside 1 to 100 (`GEOGRAPHIC_CLONE_LIMIT`, the field's bound); @@ -24,7 +24,6 @@ The UK spine gains G1 and G1b, the steps that will give it geographic clones, wi - its copy path, read from the lineage columns through `source_lineage.copy_paths`: empty for clone 0, and the `geographic_support` branch at `k` for clone `k`. So each clone keys as the dense identity keys that clone, and a larger K keeps every key. The key is in the graph's `string` dtype (`StringDtype("python")`). It refuses, by count or by name and never printing a key, a frame that already carries the key or lacks G1's int64 columns, a source year frs_release.json pins no vintage for, and repeated keys. Its evidence gives the distinct source households rounded through `receipt_counts`, that the keys are unique, the source, the vintages and the columns it reads. -- **Not wired yet (G1, G1b).** No declaration, op kind, graph node or kernel names G1 or G1b, and no build module imports them, so neither runs in a build. They move no output and no node key until they are declared. - **SPI bonuses per clone (salary sacrifice).** `draw_spi_bonuses` takes `clone_indices`, each row's geographic clone. The stage passes each person's household `household_clone_index`, and a frame without the column is one clone. - The pay cells and the donor-source floor still pool every clone's donors, the floor counted in distinct source households. - Each SPI-channel adult now draws its bonus share from the donors of its own clone in its cell, in id order, on the same person-keyed uniform. @@ -48,4 +47,4 @@ The UK spine gains G1 and G1b, the steps that will give it geographic clones, wi - `test_uk_geographic_identity.py` holds every clone's key to the dense identity kernel's at K = 1 and 3. It also checks the clone path, uniqueness and nesting across K, the dtype, the vintage for the release's survey and base years, the evidence and each refusal. - `test_uk_clone_first_nesting.py` runs an invented frame through G1 and stages 16, 17 and 42 at K = 1, 2, 3, 7 and 15. Clone 0 equals the one-clone frame on its ids, G1's columns, stage 16's copies and stage 17's seats, and every frame and copy keeps the clone-index invariant. K × each clone-0 weight matches one clone's within stated rounding classes, stage 42 nests approximately, and through G1 rule B's correction lands on a copy at 7 and 13 clones. - The salary-sacrifice and UC capital tests cover the per-clone draw and its refusal, the stage at one and three clones, a one-clone index, clones that differ, and a high-value copy. - - `test_uk_cgt_support.py` holds every spelling of the household clone index to the module's name, and its nesting scenario's data move to `test_support` for the nesting test to share. `test_uk_clone_expansion_core.py` takes G1's declaration, and `test_uk_copy_id_table.py` lists `geographic_expand` among the copy-id readers. `test_uk_graph.py` re-derives the binding tables, checks that no spine node writes the clone index columns, and checks that an edit to `geographic_clones` re-keys every counting stage. + - `test_uk_cgt_support.py` holds every spelling of the household clone index to the module's name, and its nesting scenario's data move to `test_support` for the nesting test to share. `test_uk_clone_expansion_core.py` takes G1's declaration, and `test_uk_copy_id_table.py` lists `geographic_expand` among the copy-id readers. `test_uk_graph.py` re-derives the binding tables, checks which nodes write the clone index columns, and checks that an edit to `geographic_clones` re-keys every counting stage. diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index 37701ef556..d698788d79 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -112,9 +112,9 @@ The full target compiler preserves the unreduced band-edge register, reference-p `--engine-blocks K` resolves `uk.full.measures` with one policyengine-uk engine per clone rather than one over the pool; above one, it must equal `--n-clones`. Each block's household weights are scaled to the pool's mass. In every simulation the block builds (the measure simulation, its baseline, and the annual-income and counterfactual simulations with theirs), `shareholding`, `corporate_land_value` and `consumption_shareholding` are set, at each period the block computes, to the block's own value times `c_b = f_b·S_b/S`. That is the block's pool-mass factor times its allocation-key total over the pool's, both from totals the representation records (microcosm#931). Before it sets them, the block checks that the engine's own key total is `f_b·S_b` and that the engine carries each key and the weights by one factor per year. The annual-income simulation carries other spells by the engine's own growth parameters, so every block uses the same factor. A guard records every formula each simulation runs once the shares are set, and judges policyengine-uk's whole-population formulas against the UK declaration (`microcosm.frame.adapters.policyengine_uk_population`, on the shared `engine_population` machinery). The three share formulas must be the corrected inputs at every period, and `months_since_last_birthday` and `attends_private_school` stored inputs. The engine's income and wealth deciles and relative-poverty flags are refused, and any other such formula that runs must be zero under the parameters checked. Any work in the Universal Credit rebalancing years (2026 on) is refused too, since policyengine-uk draws new-claimant status there by benefit-unit position. The measure step records a violation rather than raising it. Once the blocks ran, `engine_population_representation.exact` holds only if every block applied its correction, passed its checks and got a clean verdict, and `exact_basis` (`pool_share_injection_with_population_guard`) names that basis. The measures then equal one engine over the pool to float32 rounding, not bitwise. An inexact per-block run is not releasable. So while policyengine-uk draws new-claimant status by position, a calibration year, or a dated measure's year, of 2026 or later leaves every per-clone run inexact, and so not releasable, and refuses every batched spine call. A single block, as on the national line, is exact by construction and records no correction or guard fields. -A spine engine call whose frame carries more than one clone runs one simulation per clone, since one simulation over a cloned spine does not fit in memory. `PolicyEngineUKEngine.materialize` decides from the frame, not from a setting: a call whose household table carries `household_clone_index` with more than one clone runs its simulations in ascending clone order, and each is released before the next is built. A frame without the column, or with one clone, takes one simulation as before. A batched call's requests are judged first. A period in the rebalancing years, a refused formula, a frame without `months_since_last_birthday`, or a request for `attends_private_school`, `private_school_vat`, `baseline_private_school_vat` or `gov_tax` before stage 52 stores attendance refuses the call before any engine is built. Until then the spine's pass lets attendance run only where the three parameters the private-school VAT rate reads equal their baseline's and `private_school_vat_change` comes out zero, and only if `gov_tax` does not run. Unlike a measure block, each batch reads the pool's shares at the frame's own weights, unscaled (`c_b = S_b/S`), and is judged under the spine's pass, which raises on any violation rather than recording it, since the values come back as plain arrays. People come back in the frame's order and groups in ascending id order, as one simulation returns them. No spine frame carries `household_clone_index` yet, so every spine call takes one simulation today, and batching moves no spine value. +A spine engine call whose frame carries more than one clone runs one simulation per clone, since one simulation over a cloned spine does not fit in memory. `PolicyEngineUKEngine.materialize` decides from the frame, not from a setting: a call whose household table carries `household_clone_index` with more than one clone runs its simulations in ascending clone order, and each is released before the next is built. A frame without the column, or with one clone, takes one simulation as before. A batched call's requests are judged first. A period in the rebalancing years, a refused formula, a frame without `months_since_last_birthday`, or a request for `attends_private_school`, `private_school_vat`, `baseline_private_school_vat` or `gov_tax` before stage 52 stores attendance refuses the call before any engine is built. Until then the spine's pass lets attendance run only where the three parameters the private-school VAT rate reads equal their baseline's and `private_school_vat_change` comes out zero, and only if `gov_tax` does not run. Unlike a measure block, each batch reads the pool's shares at the frame's own weights, unscaled (`c_b = S_b/S`), and is judged under the spine's pass, which raises on any violation rather than recording it, since the values come back as plain arrays. People come back in the frame's order and groups in ascending id order, as one simulation returns them. Every spine frame from `geographic_expand` on carries `household_clone_index`, 0 throughout while the spine build runs one clone, so every spine call takes one simulation today, and batching moves no spine value. -policyengine-uk's `attends_private_school` ranks each household's market income plus benefits, less its income tax and National Insurance, over the people a simulation holds, so stage 52 (`private_school_attendance`) restates the formula's last step over the whole frame. It computes that income from household-local engine values, ranks it with microdf's `MicroSeries.percentile_rank`, and reads the attendance rates and population factor from the engine's own parameters. Over one simulation it also asks for the engine's flags, stores them once the restatement matches them bit for bit, and raises otherwise. Clone by clone it asks only for the household-local values and stores the restatement. Its evidence records `rank_route` (`engine_with_parity` or `pooled_restatement`) and `engine_parity_checked`. The spine sidecar's `rules_engine` gains `engine_batches` only on a spine whose calls run clone by clone, and the engine identity does not move with batching. Every stage that calls the engine through the adapter binds the adapter, `engine_population` and `policyengine_uk_population` into its implementation hash, and `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. +policyengine-uk's `attends_private_school` ranks each household's market income plus benefits, less its income tax and National Insurance, over the people a simulation holds, so stage 52 (`private_school_attendance`) restates the formula's last step over the whole frame. It computes that income from household-local engine values, ranks it with microdf's `MicroSeries.percentile_rank`, and reads the attendance rates and population factor from the engine's own parameters. Over one simulation it also asks for the engine's flags, stores them once the restatement matches them bit for bit, and raises otherwise. Clone by clone it asks only for the household-local values and stores the restatement. Its evidence records `rank_route` (`engine_with_parity` or `pooled_restatement`) and `engine_parity_checked`. The spine sidecar's `rules_engine` gains `engine_batches`, and beside it `microdf_python`, the microdf-python version whose weighted percentile rank `private_school_attendance` then stores, only on a spine whose calls run clone by clone, and the engine identity does not move with batching. Every stage that calls the engine through the adapter binds the adapter, `engine_population` and `policyengine_uk_population` into its implementation hash, and `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. ## Replay and outputs diff --git a/docs/uk-isa-subscriptions-1415.md b/docs/uk-isa-subscriptions-1415.md index 879ced1299..2dc184b3e4 100644 --- a/docs/uk-isa-subscriptions-1415.md +++ b/docs/uk-isa-subscriptions-1415.md @@ -82,7 +82,9 @@ How the tables relate, on HMRC's own figures: (`shortfall_weight_unmet`); no holder is invented. 3. **How much.** Within each income band and class, amounts follow Table 9.7's subscription bands: uniform inside a band, exactly £20,000 at the top. The - rank blends the person's balance rank with a keyed uniform. Holders tied on a + rank blends the person's rank on the class's balance (cash for cash only, + stocks and shares for stocks and shares only, the two together for both) + with a keyed uniform, half each (`balance_rank_weight`). Holders tied on a balance share its rank, so the keyed uniform alone orders them. 4. **The both class.** Each person's amount is split by their balance shares, shifted on the logit scale so the class's stocks-and-shares total equals @@ -91,20 +93,27 @@ How the tables relate, on HMRC's own figures: 5. **Growth to 2024-25.** Each class's people grow by Table 9.4's account growth (cash accounts for cash only, stocks-and-shares accounts for stocks and shares only, both together for both), added from eligible holders not yet - subscribing. Amounts then scale, one factor per type, to Table 9.4's 2024-25 - amounts (£95.6bn of cash and £37.2bn of stocks and shares). Subscribers - already at the £20,000 overall limit stay there. A person whose combined - amount lies between £18,000 (Table 9.7's last band floor below the limit) and - the limit moves on a line from their scaled £18,000 up to it, so the range just - below the limit stays filled; the rest are capped at it. + subscribing. The added people are selected, and their amounts drawn as in + step 3, in walks of their own, apart from the 2023-24 subscribers; those + added to the both class split at the shift step 4 found. Amounts then + scale, one factor per type, to Table 9.4's 2024-25 amounts (£95.6bn of cash + and £37.2bn of stocks and shares). Subscribers already at the £20,000 + overall limit stay there. A person whose combined amount lies between + £18,000 (Table 9.7's last band floor below the limit) and the limit moves on + a line from their scaled £18,000 up to it, so the range just below the limit + stays filled; the rest are capped at it. 6. **Lifetime ISA payments.** Every holder aged 18 to 47 with room under the - overall limit pays in; a holder with none pays nothing. 18 to 47 are the ages - that can hold one in 2024-25, since accounts opened at 18 to 39 from April - 2017. Amounts lie in (0, £4,000], drawn by a truncated-exponential tilt so - the total equals Table 9.4's 2024-25 Lifetime ISA amount (£2,818m from - 1,136,000 accounts), so the baseline leaves nothing above the overall limit. - The rank blends the person's Lifetime ISA balance rank with a keyed uniform, - as in step 3, but over every holder aged 18 to 47, not by income band. + overall limit pays in; a holder with none pays nothing, and so does a holder + over 47. 18 to 47 are the ages that can hold one in 2024-25, since accounts + opened at 18 to 39 from April 2017. Amounts lie in (0, £4,000]: each is a + truncated-exponential draw capped at the holder's room under the overall + limit, with the tilt set so the total equals Table 9.4's 2024-25 Lifetime + ISA amount (£2,818m from 1,136,000 accounts). The cap, not the tilt, is what + leaves nothing above the overall limit in the baseline. The rank blends the + person's Lifetime ISA balance rank with a keyed uniform, as in step 3 and + with step 3's `balance_rank_weight`, since the Lifetime ISA operation + declares none of its own, but over every holder aged 18 to 47, not by + income band. ## Decisions (28 October 2026) diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index 9ea9c9ffea..e05dcd8b29 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -81,6 +81,10 @@ "classify_cgt_band_facts_with_reviewed_fence", "classify_hmrc_income_facts_with_reviewed_fences", "clone_records", + # microcosm#931: a frame's geographic clones, and each household's + # geography key from its source and clone path. + "expand_geographic_clones", + "key_households_by_clone_path", "clean_nts_travel_tables", "clean_was_lisa_donor", "clean_aps_occupation_table", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 4f581481ed..919b00668f 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -10092,6 +10092,91 @@ } } }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "entity", + "clone_index_columns", + "ordinal_bound", + "id_remapping", + "order", + "weight_split", + "weight_kind_out", + "source_columns", + "conservation", + "declared_factor", + "reason" + ], + "properties": { + "kind": { + "const": "expand_geographic_clones" + }, + "entity": { + "type": "string" + }, + "clone_index_columns": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "ordinal_bound": { + "type": "integer", + "minimum": 1 + }, + "id_remapping": { + "type": "string" + }, + "order": { + "type": "string" + }, + "weight_split": { + "type": "string" + }, + "weight_kind_out": { + "type": "string" + }, + "source_columns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "conservation": { + "type": "string" + }, + "declared_factor": { + "type": "number" + }, + "reason": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "source", + "vintage_resource" + ], + "properties": { + "kind": { + "const": "key_households_by_clone_path" + }, + "source": { + "type": "string" + }, + "vintage_resource": { + "type": "string" + } + } + }, { "type": "object", "additionalProperties": false, diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index cacff20e6c..f746545fb6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -192,7 +192,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +212,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -220,6 +220,31 @@ "stage": "etb_vat", "status": "required_at_build" }, + "geographic_expand": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "person_clone_index", + "benunit_clone_index", + "household_clone_index", + "source_household_id", + "source_year", + "source_household_key" + ], + "required_mass_change_reason": "Geographic clones split each household's design weight equally among its clones, so household mass is conserved.", + "rewrites": [], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", + "source_vintages": { + "source": "Microcosm structural operator (microcosm#931); the FRS spine's own households, no publisher input", + "survey": "Family Resources Survey 2024-25" + }, + "stage": "geographic_expand", + "status": "required_at_build" + }, "high_value_home_support": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -234,7 +259,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +280,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +304,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +394,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +421,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +493,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +519,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -535,9 +560,9 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { - "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", + "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor) over every person in the frame, from the pinned engine's values and parameters", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" }, "stage": "private_school_attendance", @@ -558,7 +583,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +604,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -602,7 +627,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" @@ -626,7 +651,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +679,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +712,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +735,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +756,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -774,7 +799,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -813,7 +838,7 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "fd7e7c48138f89d231af354dd8ef396bb151ddfd374a7ec9a4828006ce148b2d", + "source_manifest_sha256": "862cf39d5d25ceb5d8df126ae9b7a7fd5f938ac0ee85727970a55ab3a9f778cc", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index d50205be04..4d3ae0d35b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -372,11 +372,13 @@ "liable_for_share_of_household_rent", "household_id", "region", + "imd_decile", "tenure_type", "accommodation_type", "num_bedrooms", "council_tax_reported", "council_tax_band", + "council_tax_band_observed", "council_tax_rebate", "council_tax_single_adult_raw", "water_and_sewerage_charges", @@ -431,7 +433,7 @@ "frs_company_car_fuel_code", "frs_company_car_contribution" ], - "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095)." + "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095). imd_decile, each household's decile on its own nation's index of multiple deprivation (IMDE, IMDW, IMDS or IMDN), 1 to 10, and 0 where none is observed or the region is unknown, and council_tax_band_observed, False in Northern Ireland, for a CTBAND code outside 1 to 9, for a band the household's region does not use and for an unknown region (council_tax_band itself is unchanged), are stage-internal carriers the clone-first spine's area assignment conditions on and the release drops; they stay out of every receipt and of the spine sidecar's nonzero shares, and a household tab without one of the four IMD variables is refused (microcosm#931)." }, { "stage": "age_tail", @@ -1064,6 +1066,377 @@ ], "notes": "Runs before BRMA and always runs; the DSA seed evaluates at the release calibration year (uk-data#478) and the modelling-start gate compares against that year, not the frame period. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted." }, + { + "stage": "frs_hmrc_spine_leaves", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", + "grain": "person", + "artifacts": [ + { + "role": "frs_table", + "table": "adult", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "adult.tab", + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "benefits", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "benefits.tab", + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "size_bytes": 2362329, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "job", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "job.tab", + "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", + "size_bytes": 10518760, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "penprov", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "penprov.tab", + "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", + "size_bytes": 513614, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "employer_rate_bands", + "kind": "public_aggregate_reference", + "resource": "ashe_employer_pension_contribution_rates.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "retain_adjudicated_frs_hmrc_leaves", + "population": "uk_frs_raw_spine", + "source_vintage": "2024-25", + "mapped_build_period": 2024, + "annualization": "weekly raw FRS amounts * 52", + "status": "adjudicated_partial_replay", + "retained_full_constituents": { + "hmrc_spi_pay": { + "spi_concept": "PAY", + "scope": "full", + "raw_sources": [ + "ADULT.INEARNS" + ], + "formula": "max(0, ADULT.INEARNS) * 52" + }, + "hmrc_spi_unemployment_benefit_income": { + "spi_concept": "UBISJA", + "scope": "full", + "raw_sources": [ + "BENEFITS.BENEFIT=14:BENAMT", + "BENEFITS.BENEFIT=19:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT in {14, 19}) * 52" + }, + "hmrc_spi_incapacity_benefit_income": { + "spi_concept": "INCPBEN", + "scope": "full", + "raw_sources": [ + "BENEFITS.BENEFIT=17:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 17) * 52", + "observed_support": "structural zero in the FRS 2023-24 and 2024-25; retained so future vintages flow" + } + }, + "retained_named_subsets": { + "ossben_identifiable_subset": { + "spi_concept": "OSSBEN", + "raw_sources": [ + "BENEFITS.BENEFIT=13:BENAMT", + "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * 52", + "scope": "identifiable_subset" + }, + "srp_regular_code5": { + "spi_concept": "SRP", + "raw_sources": [ + "BENEFITS.BENEFIT=5:BENAMT" + ], + "formula": "sum(BENAMT where BENEFIT == 5) * 52", + "scope": "regular_code5_subset" + } + }, + "source_absent_full_constituents": [ + "EPB", + "EXPS", + "TAXTERM", + "MOTHINC", + "OTHERINC" + ], + "full_concepts_forbidden_on_frs": [ + "hmrc_spi_employment_benefits", + "hmrc_spi_employment_expenses", + "hmrc_spi_taxable_termination_pay", + "hmrc_spi_miscellaneous_employment_income", + "hmrc_spi_other_income", + "hmrc_spi_other_social_security_income", + "hmrc_spi_state_pension_income" + ], + "forbid_proxy_substitution": [ + "employment_income", + "miscellaneous_income" + ], + "fail_on_missing_retained_constituent": true, + "fail_on_full_concept_alias": true + }, + { + "kind": "draw_employer_pension_contributions_from_rate_bands", + "output": "employer_pension_contributions", + "artifact_role": "employer_rate_bands", + "resource": "ashe_employer_pension_contribution_rates.json", + "scheme_source": "PENPROV.STEMPPEN 1 to 4 (employer schemes; occupational before group personal, group stakeholder, other)", + "sector_source": "JOB.JOBSECT (first job with a stated sector)", + "occupational_split": "defined benefit or defined contribution by ASHE Table P2 job counts in the sector", + "rate_basis": "full pay (employment_income)", + "top_band_ceiling": 0.3, + "seed": 0, + "salts": [ + "employer_pension_occupational_type", + "employer_pension_contribution_rate" + ] + } + ], + "outputs": [ + "hmrc_spi_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_incapacity_benefit_income", + "ossben_identifiable_subset", + "srp_regular_code5", + "employer_pension_contributions" + ], + "nonnegative_outputs": [ + "hmrc_spi_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_incapacity_benefit_income", + "ossben_identifiable_subset", + "srp_regular_code5", + "employer_pension_contributions" + ], + "notes": "Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then draws employer pension contributions: each member of an employer scheme (PENPROV.STEMPPEN 1 to 4) with earnings takes an ASHE 2024 employer contribution rate for their scheme type and sector (Table P10 band shares, full pay, all industries; occupational schemes split into defined benefit and defined contribution by the sector's Table P2 job counts), uniform within the band (top band 20% to 30%), times pay. It replaces the incumbent's three times the employee contribution (microcosm#1069 c9, R6)." + }, + { + "stage": "frs_annual_income_spells", + "survey": "Family Resources Survey 2024-25", + "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 adult, job and household tabs; the OBR average earnings index pinned in hmrc_uprating_engine_pins.json.", + "grain": "person", + "artifacts": [ + { + "role": "frs_table", + "table": "adult", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "adult.tab", + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "job", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "job.tab", + "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", + "size_bytes": 10518760, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "frs_table", + "table": "househol", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "househol.tab", + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "size_bytes": 12108606, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" + }, + { + "role": "earnings_index", + "kind": "public_aggregate_reference", + "resource": "hmrc_uprating_engine_pins.json", + "format": "json", + "runtime_sha256_required": true + } + ], + "operations": [ + { + "kind": "read_tables", + "format": "tab", + "delimiter": "\t", + "lowercase_columns": true, + "numeric_errors": "coerce", + "runtime_sha256_required": true + }, + { + "kind": "derive_annual_income_spells", + "outputs": [ + "employment_income_annual_share", + "employment_income_other_spells", + "self_employment_income_annual_share", + "self_employment_income_other_spells", + "private_pension_income_annual_share", + "contributory_benefits_annual_share" + ], + "window_months": 12, + "leaving_date_share": 0.75, + "unknown_leaving_months": 6.0, + "donor_recent_start_months": 24, + "minimum_donors": 30, + "donor_age_edges": [ + 25, + 35, + 50, + 65 + ], + "seed": 0, + "salt": "employment_income_other_spells", + "earnings_index": "gov.economic_assumptions.indices.obr.average_earnings", + "earnings_index_from": "2023-01-01", + "earnings_index_to": "2024-01-01", + "maximum_uprating_months": 24 + } + ], + "outputs": [ + "employment_income_annual_share", + "employment_income_other_spells", + "self_employment_income_annual_share", + "self_employment_income_other_spells", + "private_pension_income_annual_share", + "contributory_benefits_annual_share" + ], + "nonnegative_outputs": [ + "employment_income_annual_share", + "employment_income_other_spells", + "self_employment_income_annual_share", + "self_employment_income_other_spells", + "private_pension_income_annual_share", + "contributory_benefits_annual_share" + ], + "rewrites": [], + "notes": "Describes each FRS adult's 12 months before the interview beside the current income columns, which it never changes, so the HMRC income-band measures can read annual income while poverty, benefits and the published engine tax keep current income. HMRC counts income received over a tax year: a person who started, left or changed a job, or had a gap, sits in another HMRC band than current pay puts them in, or in none. Outputs: the share of the year paid at today's employee pay and annual pay from other spells (chiefly a recent leaver's last job), the same pair for self-employment, and the shares of the year a private pension started on retiring and contributory JSA, ESA and Incapacity Benefit started on leaving work were in payment; an annual amount is the current amount times its share plus the other spells. The State Pension needs none: policyengine-uk's months_since_state_pension_age gives its share. Months come from CURRJOBM and JOB.WRKPREV for people in work (a person in work is never a leaver), the SAMESIT/SDEMP calendar only where it records a change, and LSTYR, LSTWRK1/2 or RETIRE1 for people out of work who worked in the last 12 months (NUMJOB2 and their last-job JOB rows); a leaving date 12 months or more before the interview gives none. A leaver's pay is the last job's usual gross pay (JOB.UGROSS, asked of people out of work who last worked in the previous 12 months, in respect of that job), scaled to the DV concept by the weighted job-level UGRSPAY/UGROSS ratio on current employees who report both and uprated by the pinned OBR average earnings index for the months since leaving; otherwise a donor's weekly pay, drawn by GROSS4 weight from current employees in their first 24 months in the job within age band x sex x full- or part-time cells (sex dropped below 30 donors), on an identity-keyed uniform (seed 0, salt employment_income_other_spells). Self-employment keeps the whole year where SEJBLONG is one or more, (SEJBMTHS + 1)/12 under a year and at most the calendar's self-employed months where self-employment is the activity today (the calendar records the main activity only, so a side business beside a job keeps its whole year); a ceased self-employed last job pays SEINCAMT (else PROFIT1, a loss as zero) for its SELWKS weeks; SEINCAM2 is already uprated to the interview by the FRS, so it is not uprated again. The receipt gives each source's support as a band of distinct households, ratios only where three or more households carry them, and the exact zeros its stage gate reads." + }, + { + "artifacts": [], + "grain": "household", + "nonnegative_outputs": [ + "person_clone_index", + "benunit_clone_index", + "household_clone_index", + "source_household_id", + "source_year" + ], + "notes": "Gives every FRS household K geographic clones before any other copy step (microcosm#931), so that each clone can later take an area of its own while together they stand for the survey household once. Clone 0 is the frame's own rows and ids; clone k repeats every person, benefit unit and household with each id minted in the copy-id table's geographic_support field at ordinal k (id_remapping), and every table carries its rows' clone, 0 to K - 1, in its clone index column. Each clone carries an equal share of its household's design weight, the same float in every clone, and the frame's mass log gains one record whose reason names no clone count. Every household carries its source_household_id (its clone-0 id), source_year (the frame's year) and source_household_key, so the stages after it count true sources in survey households. K is a parameter of the stage's graph node, never of this declaration, so the declaration and its contract are the same at every K. The stage runs after frs_hmrc_spine_leaves and frs_annual_income_spells, which join the raw FRS tabs on raw person ids, and before frs_take_up, frs_person_draws, frs_household_draws and frs_brma, which run on every clone. Its evidence gives K, the copy-id field and its radix, the distinct source households rounded by the shared disclosure rules, and the weight split and kind; no row count, id or weight.", + "operations": [ + { + "clone_index_columns": { + "benunit": "benunit_clone_index", + "household": "household_clone_index", + "person": "person_clone_index" + }, + "conservation": "household_total_to_rounding", + "declared_factor": 1, + "entity": "household", + "id_remapping": "copy-id table field geographic_support; clone k takes every entity id + k x the field's radix (10**9)", + "kind": "expand_geographic_clones", + "order": "clone_major", + "ordinal_bound": 100, + "reason": "Geographic clones split each household's design weight equally among its clones, so household mass is conserved.", + "source_columns": [ + "source_household_id", + "source_year", + "source_household_key" + ], + "weight_kind_out": "design", + "weight_split": "equal" + } + ], + "outputs": [ + "person_clone_index", + "benunit_clone_index", + "household_clone_index", + "source_household_id", + "source_year", + "source_household_key" + ], + "rewrites": [], + "source": "Microcosm structural operator (microcosm#931); the FRS spine's own households, no publisher input", + "stage": "geographic_expand", + "survey": "Family Resources Survey 2024-25" + }, + { + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "frs_release.json", + "role": "frs_release", + "runtime_sha256_required": true + } + ], + "grain": "household", + "notes": "Keys every household for the area assignment (microcosm#931): geography_household_key, as the dense line's uk.full.identity writes it, from the identity source (source), the FRS vintage frs_release.json pins for the household's source_year, its source_household_id and its clone path, read from the lineage columns geographic_expand writes, never from ids, weights or values: no branch for clone 0 and the geographic_support branch at ordinal k for clone k. So clone 0 keys as the dense line keys the same base household, and growing K never changes an existing household's key. A source year the release pins no vintage for, and keys that repeat, are refused by count. The copy steps after it copy a household's key with the household. Its evidence gives the distinct source households rounded by the shared disclosure rules, that the keys are unique, the source, the vintages and the columns it reads; no key is printed.", + "operations": [ + { + "kind": "key_households_by_clone_path", + "source": "frs", + "vintage_resource": "frs_release.json" + } + ], + "outputs": [ + "geography_household_key" + ], + "rewrites": [], + "source": "Microcosm structural operator (microcosm#931); the FRS vintage frs_release.json pins, no publisher input", + "stage": "geographic_identity", + "survey": "Family Resources Survey 2024-25" + }, { "stage": "frs_take_up", "survey": "Family Resources Survey 2024-25", @@ -1238,440 +1611,144 @@ }, { "kind": "assign_binary_from_banded_rates", - "output": "would_claim_scp", - "draw": "would_claim_scp", - "band_column": "age", - "bands": [ - { - "upper_exclusive": 6, - "rate_key": "scp_under_6" - }, - { - "rate_key": "scp_6_plus" - } - ], - "seed": 0 - }, - { - "kind": "assign_uniform_draw", - "output": "attends_private_school_random_draw", - "seed": 0 - }, - { - "kind": "assign_period_constant", - "output": "tax_free_childcare_spend_routed_share", - "contract_key": "tax_free_childcare_spend_routed_share", - "period_source": "build_year", - "dtype": "float64" - }, - { - "kind": "assign_binary_from_rate", - "output": "is_apprentice", - "draw": "is_apprentice", - "rate_key": "apprentice_stock_19_to_24", - "assignment_unit": "person", - "eligibility": "employees (employment_status FT_EMPLOYED or PT_EMPLOYED, from frs_employment) aged 21 to 24 who are not in full-time education (is_in_full_time_education, from frs_education; the FRS codes no apprentice as in full-time education): SSCBA 1992 s.9A already zero-rates every earner under 21 and s.9B ends the apprentice zero rate at 25, so no other age changes employer NICs, and leaving them False keeps is_in_non_advanced_education, minimum_wage, Care to Learn and the 16 to 19 Bursary Fund unchanged; the flagged keep their age-related minimum_wage too, because policyengine-uk 2.162.8 (policyengine-uk#2683) gives the apprentice rate only under 19 or with is_in_first_year_of_apprenticeship, which the build leaves at its default (False), and the stage refuses an engine without that split or the s.9B threshold", - "rate_derivation": "the contract's apprentice_stock_19_to_24 is DfE's England apprentices in learning at the start of the academic year (participation less starts) per head aged 19 to 24, a point-in-time count taken as uniform across those ages and UK-wide; the target is r times the weighted population aged 21 to 24 at the frame's household weights, and eligible employees are flagged in the order of their identity-keyed uniform until their weight is nearest it (uc_take_up's rule), or all of them when they weigh no more than it; the receipt records the target, the flagged weight, the support bands and the relief the flags imply at the installed engine's Class 1 schedule, and uk_stage_frs_person_draws_apprentices gates them, the relief at or below HMRC's NIC - NS1", - "seed": 0 - } - ], - "outputs": [ - "would_claim_marriage_allowance", - "would_claim_scp", - "attends_private_school_random_draw", - "tax_free_childcare_spend_routed_share", - "is_apprentice" - ], - "nonnegative_outputs": [ - "attends_private_school_random_draw", - "tax_free_childcare_spend_routed_share" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. is_apprentice has no FRS source (no apprenticeship variable in the 2022-23 to 2024-25 adult tables; TRAIN2, NI3TRAIN and TRAINEE code government employment schemes) and is drawn to DfE's point-in-time rate for employees aged 21 to 24 only." - }, - { - "stage": "frs_household_draws", - "survey": "Family Resources Survey 2024-25", - "source": "Sourced UK stochastic contract rates and identity-keyed deterministic draws.", - "grain": "household", - "artifacts": [], - "operations": [ - { - "kind": "assign_binary_from_rate", - "output": "household_owns_tv", - "draw": "household_owns_tv", - "rate_key": "tv_ownership_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_evade_tv_licence_fee", - "draw": "would_evade_tv_licence_fee", - "rate_key": "tv_licence_evasion_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "main_residential_property_purchased_is_first_home", - "draw": "main_residential_property_purchased_is_first_home", - "rate_key": "first_time_buyer_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "property_purchased", - "draw": "property_purchased", - "rate_key": "property_purchase_rate", - "seed": 0 - } - ], - "outputs": [ - "household_owns_tv", - "would_evade_tv_licence_fee", - "main_residential_property_purchased_is_first_home", - "property_purchased" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share." - }, - { - "stage": "frs_brma", - "survey": "Family Resources Survey 2024-25", - "source": "Census private-rented households by region, BRMA and bedrooms (ONS and NISRA 2021, NRS 2022) and policyengine-uk LHA_category predictor.", - "grain": "household", - "artifacts": [ - { - "role": "count_resource", - "resource": "brma_private_rented_households.json", - "kind": "public_aggregated_counts", - "format": "json" - } - ], - "operations": [ - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "LHA_category" - ], - "consumed_only": true - }, - { - "kind": "sample_categorical_from_count_table", - "output": "brma", - "count_resource": "brma_private_rented_households.json", - "cell_columns": [ - "region", - "LHA_category" - ], - "assignment_grain": "benunit", - "collapse_to": "household", - "collapse_method": "uniform_member_pick", - "seed": 0 - } - ], - "outputs": [ - "brma" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick." - }, - { - "stage": "frs_hmrc_spine_leaves", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", - "grain": "person", - "artifacts": [ - { - "role": "frs_table", - "table": "adult", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "adult.tab", - "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", - "size_bytes": 34885825, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "benefits", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "benefits.tab", - "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", - "size_bytes": 2362329, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "job", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "job.tab", - "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", - "size_bytes": 10518760, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "penprov", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "penprov.tab", - "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", - "size_bytes": 513614, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "employer_rate_bands", - "kind": "public_aggregate_reference", - "resource": "ashe_employer_pension_contribution_rates.json", - "format": "json", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "retain_adjudicated_frs_hmrc_leaves", - "population": "uk_frs_raw_spine", - "source_vintage": "2024-25", - "mapped_build_period": 2024, - "annualization": "weekly raw FRS amounts * 52", - "status": "adjudicated_partial_replay", - "retained_full_constituents": { - "hmrc_spi_pay": { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": [ - "ADULT.INEARNS" - ], - "formula": "max(0, ADULT.INEARNS) * 52" - }, - "hmrc_spi_unemployment_benefit_income": { - "spi_concept": "UBISJA", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=14:BENAMT", - "BENEFITS.BENEFIT=19:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT in {14, 19}) * 52" - }, - "hmrc_spi_incapacity_benefit_income": { - "spi_concept": "INCPBEN", - "scope": "full", - "raw_sources": [ - "BENEFITS.BENEFIT=17:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 17) * 52", - "observed_support": "structural zero in the FRS 2023-24 and 2024-25; retained so future vintages flow" - } - }, - "retained_named_subsets": { - "ossben_identifiable_subset": { - "spi_concept": "OSSBEN", - "raw_sources": [ - "BENEFITS.BENEFIT=13:BENAMT", - "BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * 52", - "scope": "identifiable_subset" + "output": "would_claim_scp", + "draw": "would_claim_scp", + "band_column": "age", + "bands": [ + { + "upper_exclusive": 6, + "rate_key": "scp_under_6" }, - "srp_regular_code5": { - "spi_concept": "SRP", - "raw_sources": [ - "BENEFITS.BENEFIT=5:BENAMT" - ], - "formula": "sum(BENAMT where BENEFIT == 5) * 52", - "scope": "regular_code5_subset" + { + "rate_key": "scp_6_plus" } - }, - "source_absent_full_constituents": [ - "EPB", - "EXPS", - "TAXTERM", - "MOTHINC", - "OTHERINC" - ], - "full_concepts_forbidden_on_frs": [ - "hmrc_spi_employment_benefits", - "hmrc_spi_employment_expenses", - "hmrc_spi_taxable_termination_pay", - "hmrc_spi_miscellaneous_employment_income", - "hmrc_spi_other_income", - "hmrc_spi_other_social_security_income", - "hmrc_spi_state_pension_income" ], - "forbid_proxy_substitution": [ - "employment_income", - "miscellaneous_income" - ], - "fail_on_missing_retained_constituent": true, - "fail_on_full_concept_alias": true + "seed": 0 }, { - "kind": "draw_employer_pension_contributions_from_rate_bands", - "output": "employer_pension_contributions", - "artifact_role": "employer_rate_bands", - "resource": "ashe_employer_pension_contribution_rates.json", - "scheme_source": "PENPROV.STEMPPEN 1 to 4 (employer schemes; occupational before group personal, group stakeholder, other)", - "sector_source": "JOB.JOBSECT (first job with a stated sector)", - "occupational_split": "defined benefit or defined contribution by ASHE Table P2 job counts in the sector", - "rate_basis": "full pay (employment_income)", - "top_band_ceiling": 0.3, - "seed": 0, - "salts": [ - "employer_pension_occupational_type", - "employer_pension_contribution_rate" - ] + "kind": "assign_uniform_draw", + "output": "attends_private_school_random_draw", + "seed": 0 + }, + { + "kind": "assign_period_constant", + "output": "tax_free_childcare_spend_routed_share", + "contract_key": "tax_free_childcare_spend_routed_share", + "period_source": "build_year", + "dtype": "float64" + }, + { + "kind": "assign_binary_from_rate", + "output": "is_apprentice", + "draw": "is_apprentice", + "rate_key": "apprentice_stock_19_to_24", + "assignment_unit": "person", + "eligibility": "employees (employment_status FT_EMPLOYED or PT_EMPLOYED, from frs_employment) aged 21 to 24 who are not in full-time education (is_in_full_time_education, from frs_education; the FRS codes no apprentice as in full-time education): SSCBA 1992 s.9A already zero-rates every earner under 21 and s.9B ends the apprentice zero rate at 25, so no other age changes employer NICs, and leaving them False keeps is_in_non_advanced_education, minimum_wage, Care to Learn and the 16 to 19 Bursary Fund unchanged; the flagged keep their age-related minimum_wage too, because policyengine-uk 2.162.8 (policyengine-uk#2683) gives the apprentice rate only under 19 or with is_in_first_year_of_apprenticeship, which the build leaves at its default (False), and the stage refuses an engine without that split or the s.9B threshold", + "rate_derivation": "the contract's apprentice_stock_19_to_24 is DfE's England apprentices in learning at the start of the academic year (participation less starts) per head aged 19 to 24, a point-in-time count taken as uniform across those ages and UK-wide; the target is r times the weighted population aged 21 to 24 at the frame's household weights, and eligible employees are flagged in the order of their identity-keyed uniform until their weight is nearest it (uc_take_up's rule), or all of them when they weigh no more than it; the receipt records the target, the flagged weight, the support bands and the relief the flags imply at the installed engine's Class 1 schedule, and uk_stage_frs_person_draws_apprentices gates them, the relief at or below HMRC's NIC - NS1", + "seed": 0 } ], "outputs": [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", - "ossben_identifiable_subset", - "srp_regular_code5", - "employer_pension_contributions" + "would_claim_marriage_allowance", + "would_claim_scp", + "attends_private_school_random_draw", + "tax_free_childcare_spend_routed_share", + "is_apprentice" ], "nonnegative_outputs": [ - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", - "ossben_identifiable_subset", - "srp_regular_code5", - "employer_pension_contributions" + "attends_private_school_random_draw", + "tax_free_childcare_spend_routed_share" ], - "notes": "Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then draws employer pension contributions: each member of an employer scheme (PENPROV.STEMPPEN 1 to 4) with earnings takes an ASHE 2024 employer contribution rate for their scheme type and sector (Table P10 band shares, full pay, all industries; occupational schemes split into defined benefit and defined contribution by the sector's Table P2 job counts), uniform within the band (top band 20% to 30%), times pay. It replaces the incumbent's three times the employee contribution (microcosm#1069 c9, R6)." + "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. is_apprentice has no FRS source (no apprenticeship variable in the 2022-23 to 2024-25 adult tables; TRAIN2, NI3TRAIN and TRAINEE code government employment schemes) and is drawn to DfE's point-in-time rate for employees aged 21 to 24 only." }, { - "stage": "frs_annual_income_spells", + "stage": "frs_household_draws", "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 adult, job and household tabs; the OBR average earnings index pinned in hmrc_uprating_engine_pins.json.", - "grain": "person", - "artifacts": [ + "source": "Sourced UK stochastic contract rates and identity-keyed deterministic draws.", + "grain": "household", + "artifacts": [], + "operations": [ { - "role": "frs_table", - "table": "adult", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "adult.tab", - "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", - "size_bytes": 34885825, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" + "kind": "assign_binary_from_rate", + "output": "household_owns_tv", + "draw": "household_owns_tv", + "rate_key": "tv_ownership_rate", + "seed": 0 }, { - "role": "frs_table", - "table": "job", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "job.tab", - "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", - "size_bytes": 10518760, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" + "kind": "assign_binary_from_rate", + "output": "would_evade_tv_licence_fee", + "draw": "would_evade_tv_licence_fee", + "rate_key": "tv_licence_evasion_rate", + "seed": 0 }, { - "role": "frs_table", - "table": "househol", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "househol.tab", - "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", - "size_bytes": 12108606, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" + "kind": "assign_binary_from_rate", + "output": "main_residential_property_purchased_is_first_home", + "draw": "main_residential_property_purchased_is_first_home", + "rate_key": "first_time_buyer_rate", + "seed": 0 }, { - "role": "earnings_index", - "kind": "public_aggregate_reference", - "resource": "hmrc_uprating_engine_pins.json", - "format": "json", - "runtime_sha256_required": true + "kind": "assign_binary_from_rate", + "output": "property_purchased", + "draw": "property_purchased", + "rate_key": "property_purchase_rate", + "seed": 0 + } + ], + "outputs": [ + "household_owns_tv", + "would_evade_tv_licence_fee", + "main_residential_property_purchased_is_first_home", + "property_purchased" + ], + "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share." + }, + { + "stage": "frs_brma", + "survey": "Family Resources Survey 2024-25", + "source": "Census private-rented households by region, BRMA and bedrooms (ONS and NISRA 2021, NRS 2022) and policyengine-uk LHA_category predictor.", + "grain": "household", + "artifacts": [ + { + "role": "count_resource", + "resource": "brma_private_rented_households.json", + "kind": "public_aggregated_counts", + "format": "json" } ], "operations": [ { - "kind": "read_tables", - "format": "tab", - "delimiter": "\t", - "lowercase_columns": true, - "numeric_errors": "coerce", - "runtime_sha256_required": true + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "LHA_category" + ], + "consumed_only": true }, { - "kind": "derive_annual_income_spells", - "outputs": [ - "employment_income_annual_share", - "employment_income_other_spells", - "self_employment_income_annual_share", - "self_employment_income_other_spells", - "private_pension_income_annual_share", - "contributory_benefits_annual_share" - ], - "window_months": 12, - "leaving_date_share": 0.75, - "unknown_leaving_months": 6.0, - "donor_recent_start_months": 24, - "minimum_donors": 30, - "donor_age_edges": [ - 25, - 35, - 50, - 65 + "kind": "sample_categorical_from_count_table", + "output": "brma", + "count_resource": "brma_private_rented_households.json", + "cell_columns": [ + "region", + "LHA_category" ], - "seed": 0, - "salt": "employment_income_other_spells", - "earnings_index": "gov.economic_assumptions.indices.obr.average_earnings", - "earnings_index_from": "2023-01-01", - "earnings_index_to": "2024-01-01", - "maximum_uprating_months": 24 + "assignment_grain": "benunit", + "collapse_to": "household", + "collapse_method": "uniform_member_pick", + "seed": 0 } ], "outputs": [ - "employment_income_annual_share", - "employment_income_other_spells", - "self_employment_income_annual_share", - "self_employment_income_other_spells", - "private_pension_income_annual_share", - "contributory_benefits_annual_share" - ], - "nonnegative_outputs": [ - "employment_income_annual_share", - "employment_income_other_spells", - "self_employment_income_annual_share", - "self_employment_income_other_spells", - "private_pension_income_annual_share", - "contributory_benefits_annual_share" + "brma" ], - "rewrites": [], - "notes": "Describes each FRS adult's 12 months before the interview beside the current income columns, which it never changes, so the HMRC income-band measures can read annual income while poverty, benefits and the published engine tax keep current income. HMRC counts income received over a tax year: a person who started, left or changed a job, or had a gap, sits in another HMRC band than current pay puts them in, or in none. Outputs: the share of the year paid at today's employee pay and annual pay from other spells (chiefly a recent leaver's last job), the same pair for self-employment, and the shares of the year a private pension started on retiring and contributory JSA, ESA and Incapacity Benefit started on leaving work were in payment; an annual amount is the current amount times its share plus the other spells. The State Pension needs none: policyengine-uk's months_since_state_pension_age gives its share. Months come from CURRJOBM and JOB.WRKPREV for people in work (a person in work is never a leaver), the SAMESIT/SDEMP calendar only where it records a change, and LSTYR, LSTWRK1/2 or RETIRE1 for people out of work who worked in the last 12 months (NUMJOB2 and their last-job JOB rows); a leaving date 12 months or more before the interview gives none. A leaver's pay is the last job's usual gross pay (JOB.UGROSS, asked of people out of work who last worked in the previous 12 months, in respect of that job), scaled to the DV concept by the weighted job-level UGRSPAY/UGROSS ratio on current employees who report both and uprated by the pinned OBR average earnings index for the months since leaving; otherwise a donor's weekly pay, drawn by GROSS4 weight from current employees in their first 24 months in the job within age band x sex x full- or part-time cells (sex dropped below 30 donors), on an identity-keyed uniform (seed 0, salt employment_income_other_spells). Self-employment keeps the whole year where SEJBLONG is one or more, (SEJBMTHS + 1)/12 under a year and at most the calendar's self-employed months where self-employment is the activity today (the calendar records the main activity only, so a side business beside a job keeps its whole year); a ceased self-employed last job pays SEINCAMT (else PROFIT1, a loss as zero) for its SELWKS weeks; SEINCAM2 is already uprated to the interview by the FRS, so it is not uprated again. The receipt gives each source's support as a band of distinct households, ratios only where three or more households carry them, and the exact zeros its stage gate reads." + "notes": "Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick." }, { "stage": "spi_support_channel", "survey": "Family Resources Survey 2024-25", - "source": "Synthetic SPI support channel drawn from the raw FRS spine as a keyed Bernoulli share, each household on its own identity-keyed uniform.", + "source": "Synthetic SPI support channel drawn from every geographic clone of the raw FRS spine's households as a keyed Bernoulli share, each household on its own identity-keyed uniform.", "grain": "household", "artifacts": [], "operations": [ @@ -1728,10 +1805,7 @@ "benunit_source_id", "household_support_channel", "household_support_clone_index", - "household_source_id", - "source_household_id", - "source_year", - "source_household_key" + "household_source_id" ], "nonnegative_outputs": [ "person_support_clone_index", @@ -1739,11 +1813,9 @@ "benunit_support_clone_index", "benunit_source_id", "household_support_clone_index", - "household_source_id", - "source_household_id", - "source_year" + "household_source_id" ], - "notes": "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's draw depends on its own id alone. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." + "notes": "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's uniform depends on its own id and the stage's seed and salt alone, and whether it is copied also on that probability. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." }, { "stage": "spi_income_band_donors", @@ -1791,11 +1863,11 @@ "minimum_donors_per_band": 120, "maximum_donor_weight": 250.0, "count_rule": "max(minimum_donors_per_band, ceil(published band taxpayers / maximum_donor_weight))", - "weight_rule": "published band taxpayers / (geographic clone count x band donor count), equal within a band", + "weight_rule": "published band taxpayers / (geographic clone count x one clone's band donor count), equal within a band", "expected_band_count": 4, "candidate_population": "raw FRS base channel claimants and partners (is_uc_claimant) aged 16 and over", "candidate_order": "person_id ascending breaks equal draw keys", - "draw": "weighted_without_replacement_identity_keyed within each geographic clone (one before uk.full.expand): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", + "draw": "weighted_without_replacement_identity_keyed within each geographic clone (a spine without household_clone_index is one clone): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", "propensity": "SPI 2022-23 FACT-weighted band share by region, sex and age band", "tape_preparation": "as hmrc_spi_income_spine prepares the tape: its stage-1 seed and draw_spi_donor_ages_by_population", "tape_preparation_seed": 42, @@ -1841,7 +1913,7 @@ "nonnegative_outputs": [ "spi_income_band_donor_lower_bound" ], - "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, with the seats counted in distinct source households, each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves)." + "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, and every clone takes the same seats, sized against the eligible households counted in distinct source households. Each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves)." }, { "stage": "hmrc_spi_income_spine", @@ -3908,7 +3980,7 @@ "stratum_lower": 0.95, "copies": 6, "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", - "copy_rule": "every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", + "copy_rule": "every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", "flag_column": "household_is_high_value_support_copy", "copies_column": "high_value_support_copies", @@ -6858,7 +6930,7 @@ "frs_benunit_capital", "would_claim_uc" ], - "notes": "Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donors widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (María's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base." + "notes": "Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donor survey records (a unit's geographic clones count once, every other copy as a record of its own) widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (María's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base." }, { "stage": "uc_limited_capability", @@ -7406,7 +7478,7 @@ ], "selection": "within each income band walk households in investable wealth descending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "maximum_copy_weight": 60.0, - "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make", "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "flag_column": "household_is_cgt_support_copy", "copies_column": "cgt_support_copies", @@ -7426,7 +7498,7 @@ "cgt_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then count that household as a family the stage never made, which the bracketed family test does not report, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." }, { "stage": "cgt_incidence_clone", @@ -8107,7 +8179,7 @@ "candidates": "a bonus (frs_bonus_amount, or an SPI-channel adult's drawn bonus) > 0 and employment_income at least the anchor's annual upper earnings limit, after the salary conversion", "move": "the bonus, at most move_pay_share_cap of employment_income, to pension_contributions_via_bonus_sacrifice; employment_income unchanged, since a sacrificed bonus is never received", "move_pay_share_cap": 0.5, - "spi_bonus_draw": "an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell, by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below", + "spi_bonus_draw": "an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell and the same geographic clone (household_clone_index; a spine without it is one clone), by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles over every clone at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below, and the lowest the one above", "spi_cell_quantiles": [ 0.1, 0.2, @@ -8149,7 +8221,7 @@ "employee_pension_contributions", "employment_income" ], - "notes": "The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household's members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin's answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR's 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (María's ruling of 2026-10-24 on Vahid's review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult's FRS bonus is its twin's, beside the tape's pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell instead." + "notes": "The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household's members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin's answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR's 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (María's ruling of 2026-10-24 on Vahid's review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult's FRS bonus is its twin's, beside the tape's pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell, in its own geographic clone, instead." }, { "stage": "carers_allowance_take_up", @@ -8374,12 +8446,12 @@ "maximum_extended_childcare_hours_usage", "max_free_entitlement_hours_used" ], - "notes": "Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units before any copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id." + "notes": "Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units, each geographic clone's on its own ids (microcosm#931), before any other copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units, every geographic clone's included, keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id." }, { "stage": "private_school_attendance", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights", - "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", + "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor) over every person in the frame, from the pinned engine's values and parameters", "grain": "person", "artifacts": [ { @@ -8409,7 +8481,7 @@ "outputs": [ "attends_private_school" ], - "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine's attendance once, at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula's last step over every person in the frame, from the engine's own values at the calibration year, each decided by a household's own records, and its own parameters: household market income plus benefits less the members' income tax and National Insurance, each household's weight once per member, microdf's weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine before its geographic clones exist), the stage stores the engine's flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine's flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine." + "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage computes the formula's attendance once, over the whole frame at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula's last step over every person in the frame, from the engine's own values at the calibration year, each decided by a household's own records, and its own parameters: household market income plus benefits less the members' income tax and National Insurance, each household's weight once per member, microdf's weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine of one geographic clone), the stage stores the engine's flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine's flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine." }, { "artifacts": [ @@ -8455,7 +8527,7 @@ "stocks_and_shares_isa_subscriptions", "lifetime_isa_payments" ], - "notes": "Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household's was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7's income bands, Table 9.8's age and sex and Table 9.9's regions and selected by keyed walks, the both class first. Amounts follow Table 9.7's subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person's amount by their balance shares, shifted so the class's stocks-and-shares total is Table 9.4's 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. People grow to 2024-25 by Table 9.4's account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4's 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7's last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (María's rulings of 29 October 2026): Table 9.7's cash bands sit above Table 9.4, and growing that excess by Table 9.4's ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4's 2024-25 Lifetime ISA amount. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.", + "notes": "Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household's was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7's income bands, Table 9.8's age and sex and Table 9.9's regions and selected by keyed walks, the both class first. Amounts follow Table 9.7's subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person's amount by their balance shares, shifted so the class's stocks-and-shares total is Table 9.4's 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. Amounts go in the order of a key that blends each subscriber's weighted rank on the class's balance (cash, stocks and shares, or the two together) with a keyed uniform at balance_rank_weight; tied balances share one weighted mid-rank, so the keyed uniform alone orders them. People grow to 2024-25 by Table 9.4's account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4's 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7's last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (María's rulings of 29 October 2026): Table 9.7's cash bands sit above Table 9.4, and growing that excess by Table 9.4's ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4's 2024-25 Lifetime ISA amount and ordered by the same blend, on the Lifetime ISA balance over every holder aged 18 to 47, with tied balances again sharing one mid-rank; the Lifetime ISA operation declares no rank weight and reuses the amounts operation's balance_rank_weight. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.", "operations": [ { "artifact_role": "isa_statistics_facts", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 9d6766d069..e35b4b7b56 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -565,11 +565,13 @@ stages: - liable_for_share_of_household_rent - household_id - region + - imd_decile - tenure_type - accommodation_type - num_bedrooms - council_tax_reported - council_tax_band + - council_tax_band_observed - council_tax_rebate - council_tax_single_adult_raw - water_and_sewerage_charges @@ -622,7 +624,7 @@ stages: - frs_company_car_price_band - frs_company_car_fuel_code - frs_company_car_contribution - notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095). + notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095). imd_decile, each household's decile on its own nation's index of multiple deprivation (IMDE, IMDW, IMDS or IMDN), 1 to 10, and 0 where none is observed or the region is unknown, and council_tax_band_observed, False in Northern Ireland, for a CTBAND code outside 1 to 9, for a band the household's region does not use and for an unknown region (council_tax_band itself is unchanged), are stage-internal carriers the clone-first spine's area assignment conditions on and the release drops; they stay out of every receipt and of the spine sidecar's nonzero shares, and a household tab without one of the four IMD variables is refused (microcosm#931). - stage: age_tail survey: Family Resources Survey 2024-25 and ONS mid-year population estimates source: Office for National Statistics mid-year population estimates for the 80+ age bands by sex, via the chronicle ledger facts the calibration register binds (target family ons_population). @@ -1049,226 +1051,6 @@ stages: nonnegative_outputs: - disabled_students_allowance_eligible_expenses notes: Runs before BRMA and always runs; the DSA seed evaluates at the release calibration year (uk-data#478) and the modelling-start gate compares against that year, not the frame period. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted. -- stage: frs_take_up - survey: Family Resources Survey 2024-25 - source: Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates. - grain: benunit - artifacts: [] - operations: - - kind: aggregate_person_to_benunit - method: any_positive - consumed_only: true - aggregates: - child_benefit_reported_anchor: child_benefit_reported - pension_credit_reported_anchor: pension_credit_reported - universal_credit_reported_anchor: universal_credit_reported - - kind: aggregate_person_to_benunit - method: any_uc_claimant_aged_18_under_pension_credit_age - consumed_only: true - aggregates: - uc_age_eligible: age - role_column: is_uc_claimant - - kind: assign_binary_with_anchored_residual - output: would_claim_child_benefit - draw: would_claim_child_benefit - rate_key: child_benefit - anchor: child_benefit_reported_anchor - seed: 0 - - kind: assign_binary_from_rate - output: child_benefit_opts_out - draw: child_benefit_opts_out - rate_key: child_benefit_opts_out_rate - seed: 0 - - kind: assign_binary_with_anchored_residual - output: would_claim_pc - draw: would_claim_pc - rate_key: pension_credit - anchor: pension_credit_reported_anchor - seed: 0 - - kind: assign_binary_with_anchored_residual - output: would_claim_uc - draw: would_claim_uc - rate_key: universal_credit - anchor: universal_credit_reported_anchor - population: uc_age_eligible - seed: 0 - - kind: assign_binary_from_rate - output: would_claim_tfc - draw: would_claim_tfc - rate_key: tax_free_childcare - seed: 0 - - kind: assign_binary_from_rate - output: would_claim_extended_childcare - draw: would_claim_extended_childcare - rate_key: extended_childcare - seed: 0 - - kind: assign_binary_from_rate - output: would_claim_universal_childcare - draw: would_claim_universal_childcare - rate_key: universal_childcare - nested_under: would_claim_extended_childcare - rate_derivation: the contract's universal_childcare rate is the rate among benefit units not claiming the extended hours; would_claim_universal_childcare = would_claim_extended_childcare OR u < rate, so every unit claiming the extended hours claims the universal hours - seed: 0 - - kind: assign_binary_from_rate - output: would_claim_targeted_childcare - draw: would_claim_targeted_childcare - rate_key: targeted_childcare - seed: 0 - - kind: assign_binary_from_banded_rates - output: would_claim_uc_childcare - draw: would_claim_uc_childcare - band_column: is_married - bands: - - upper_exclusive: 1 - rate_key: uc_childcare_single - - rate_key: uc_childcare_couple - seed: 0 - - kind: assign_clipped_normal - output: maximum_extended_childcare_hours_usage - draw: maximum_extended_childcare_hours_usage - distribution_key: maximum_extended_childcare_hours_usage - seed: 0 - - kind: assign_binary_from_rate - output: max_free_entitlement_hours_used - draw: max_free_entitlement_hours_used - rate_key: extended_childcare_age_0_usage - eligibility: persons of DfE age 0 in benefit units that claim the working-parent hours (would_claim_extended_childcare) - assignment_unit: person - rate_derivation: a child of DfE age 0 in a claiming unit uses the working-parent hours where u < its age's rate and gets max_free_entitlement_hours_used 0 otherwise; every other person keeps the engine's default of 30; one draw per person serves both ages' operations - seed: 0 - - kind: assign_binary_from_rate - output: max_free_entitlement_hours_used - draw: max_free_entitlement_hours_used - rate_key: extended_childcare_age_1_usage - eligibility: persons of DfE age 1 in benefit units that claim the working-parent hours (would_claim_extended_childcare) - assignment_unit: person - rate_derivation: a child of DfE age 1 in a claiming unit uses the working-parent hours where u < its age's rate and gets max_free_entitlement_hours_used 0 otherwise; every other person keeps the engine's default of 30; one draw per person serves both ages' operations - seed: 0 - outputs: - - would_claim_child_benefit - - child_benefit_opts_out - - would_claim_pc - - would_claim_uc - - would_claim_tfc - - would_claim_extended_childcare - - would_claim_universal_childcare - - would_claim_targeted_childcare - - would_claim_uc_childcare - - maximum_extended_childcare_hours_usage - - max_free_entitlement_hours_used - nonnegative_outputs: - - maximum_extended_childcare_hours_usage - - max_free_entitlement_hours_used - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. The Universal Credit draw's population is the benefit units with a UC claimant or partner (is_uc_claimant) aged 18 or over and under Pension Credit qualifying age (66 for everyone who can reach it in this release), the age and role conditions of policyengine-uk 2.177.7's is_uc_eligible; the engine lockstep test pins it and names the engine cases not mirrored (16- and 17-year-old claimants under the UC Regs 2013 reg 8 exceptions, mixed-age couples on the Pension Credit route, from 2.128.0 pension-age families moved onto Universal Credit when their legacy benefit closes, SI 2014/1230 reg 60A, and from 2.173.2 a unit whose working-age claimants all receive education with no exception, since frs_education sets is_in_full_time_education from the FRS). So the contract rate is a share of those units, an anchor outside them stays true, and any other unit is never drawn; a unit with no claimant or partner of that age keeps its reported Housing Benefit, which a claim it cannot make no longer zeroes (#882, microcosm#867, microcosm#1095). The universal childcare flag is nested under the working-parent one. A unit that claims the extended hours claims the universal hours, and any other unit claims them at the contract's universal_childcare rate, the rate conditional on not claiming the extended hours; the extended draw is unchanged. childcare_copy_take_up later re-draws the childcare flags and hours of the support and CGT copies on their own benefit units. A child of DfE age 0 or 1 (its age on 31 December of the fit's model year, the age DfE counts by) in a unit that claims the working-parent hours uses them at its age's contract rate, extended_childcare_age_0_usage or extended_childcare_age_1_usage (DfE registers far fewer of the youngest eligible children than of those aged 2 to 4, and fewer at 9 to 11 months than at age 1); a child that does not gets max_free_entitlement_hours_used 0, so the engine funds it no hours, and every other person keeps the engine's default of 30. -- stage: frs_person_draws - survey: Family Resources Survey 2024-25 - source: Sourced UK take-up contract rates and identity-keyed deterministic draws. - grain: person - artifacts: - - role: stochastic_contract - resource: take_up_contract.json - kind: public_aggregate_reference - format: json - operations: - - kind: assign_binary_from_rate - output: would_claim_marriage_allowance - draw: would_claim_marriage_allowance - rate_key: marriage_allowance - seed: 0 - - kind: assign_binary_from_banded_rates - output: would_claim_scp - draw: would_claim_scp - band_column: age - bands: - - upper_exclusive: 6 - rate_key: scp_under_6 - - rate_key: scp_6_plus - seed: 0 - - kind: assign_uniform_draw - output: attends_private_school_random_draw - seed: 0 - - kind: assign_period_constant - output: tax_free_childcare_spend_routed_share - contract_key: tax_free_childcare_spend_routed_share - period_source: build_year - dtype: float64 - - kind: assign_binary_from_rate - output: is_apprentice - draw: is_apprentice - rate_key: apprentice_stock_19_to_24 - assignment_unit: person - eligibility: 'employees (employment_status FT_EMPLOYED or PT_EMPLOYED, from frs_employment) aged 21 to 24 who are not in full-time education (is_in_full_time_education, from frs_education; the FRS codes no apprentice as in full-time education): SSCBA 1992 s.9A already zero-rates every earner under 21 and s.9B ends the apprentice zero rate at 25, so no other age changes employer NICs, and leaving them False keeps is_in_non_advanced_education, minimum_wage, Care to Learn and the 16 to 19 Bursary Fund unchanged; the flagged keep their age-related minimum_wage too, because policyengine-uk 2.162.8 (policyengine-uk#2683) gives the apprentice rate only under 19 or with is_in_first_year_of_apprenticeship, which the build leaves at its default (False), and the stage refuses an engine without that split or the s.9B threshold' - rate_derivation: 'the contract''s apprentice_stock_19_to_24 is DfE''s England apprentices in learning at the start of the academic year (participation less starts) per head aged 19 to 24, a point-in-time count taken as uniform across those ages and UK-wide; the target is r times the weighted population aged 21 to 24 at the frame''s household weights, and eligible employees are flagged in the order of their identity-keyed uniform until their weight is nearest it (uc_take_up''s rule), or all of them when they weigh no more than it; the receipt records the target, the flagged weight, the support bands and the relief the flags imply at the installed engine''s Class 1 schedule, and uk_stage_frs_person_draws_apprentices gates them, the relief at or below HMRC''s NIC - NS1' - seed: 0 - outputs: - - would_claim_marriage_allowance - - would_claim_scp - - attends_private_school_random_draw - - tax_free_childcare_spend_routed_share - - is_apprentice - nonnegative_outputs: - - attends_private_school_random_draw - - tax_free_childcare_spend_routed_share - notes: 'Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. is_apprentice has no FRS source (no apprenticeship variable in the 2022-23 to 2024-25 adult tables; TRAIN2, NI3TRAIN and TRAINEE code government employment schemes) and is drawn to DfE''s point-in-time rate for employees aged 21 to 24 only.' -- stage: frs_household_draws - survey: Family Resources Survey 2024-25 - source: Sourced UK stochastic contract rates and identity-keyed deterministic draws. - grain: household - artifacts: [] - operations: - - kind: assign_binary_from_rate - output: household_owns_tv - draw: household_owns_tv - rate_key: tv_ownership_rate - seed: 0 - - kind: assign_binary_from_rate - output: would_evade_tv_licence_fee - draw: would_evade_tv_licence_fee - rate_key: tv_licence_evasion_rate - seed: 0 - - kind: assign_binary_from_rate - output: main_residential_property_purchased_is_first_home - draw: main_residential_property_purchased_is_first_home - rate_key: first_time_buyer_rate - seed: 0 - - kind: assign_binary_from_rate - output: property_purchased - draw: property_purchased - rate_key: property_purchase_rate - seed: 0 - outputs: - - household_owns_tv - - would_evade_tv_licence_fee - - main_residential_property_purchased_is_first_home - - property_purchased - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share. -- stage: frs_brma - survey: Family Resources Survey 2024-25 - source: Census private-rented households by region, BRMA and bedrooms (ONS and NISRA 2021, NRS 2022) and policyengine-uk LHA_category predictor. - grain: household - artifacts: - - role: count_resource - resource: brma_private_rented_households.json - kind: public_aggregated_counts - format: json - operations: - - kind: materialize_rules_engine_predictors - predictors: - - LHA_category - consumed_only: true - - kind: sample_categorical_from_count_table - output: brma - count_resource: brma_private_rented_households.json - cell_columns: - - region - - LHA_category - assignment_grain: benunit - collapse_to: household - collapse_method: uniform_member_pick - seed: 0 - outputs: - - brma - notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick. - stage: frs_hmrc_spine_leaves survey: Family Resources Survey 2024-25 source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input @@ -1509,9 +1291,286 @@ stages: - contributory_benefits_annual_share rewrites: [] notes: 'Describes each FRS adult''s 12 months before the interview beside the current income columns, which it never changes, so the HMRC income-band measures can read annual income while poverty, benefits and the published engine tax keep current income. HMRC counts income received over a tax year: a person who started, left or changed a job, or had a gap, sits in another HMRC band than current pay puts them in, or in none. Outputs: the share of the year paid at today''s employee pay and annual pay from other spells (chiefly a recent leaver''s last job), the same pair for self-employment, and the shares of the year a private pension started on retiring and contributory JSA, ESA and Incapacity Benefit started on leaving work were in payment; an annual amount is the current amount times its share plus the other spells. The State Pension needs none: policyengine-uk''s months_since_state_pension_age gives its share. Months come from CURRJOBM and JOB.WRKPREV for people in work (a person in work is never a leaver), the SAMESIT/SDEMP calendar only where it records a change, and LSTYR, LSTWRK1/2 or RETIRE1 for people out of work who worked in the last 12 months (NUMJOB2 and their last-job JOB rows); a leaving date 12 months or more before the interview gives none. A leaver''s pay is the last job''s usual gross pay (JOB.UGROSS, asked of people out of work who last worked in the previous 12 months, in respect of that job), scaled to the DV concept by the weighted job-level UGRSPAY/UGROSS ratio on current employees who report both and uprated by the pinned OBR average earnings index for the months since leaving; otherwise a donor''s weekly pay, drawn by GROSS4 weight from current employees in their first 24 months in the job within age band x sex x full- or part-time cells (sex dropped below 30 donors), on an identity-keyed uniform (seed 0, salt employment_income_other_spells). Self-employment keeps the whole year where SEJBLONG is one or more, (SEJBMTHS + 1)/12 under a year and at most the calendar''s self-employed months where self-employment is the activity today (the calendar records the main activity only, so a side business beside a job keeps its whole year); a ceased self-employed last job pays SEINCAMT (else PROFIT1, a loss as zero) for its SELWKS weeks; SEINCAM2 is already uprated to the interview by the FRS, so it is not uprated again. The receipt gives each source''s support as a band of distinct households, ratios only where three or more households carry them, and the exact zeros its stage gate reads.' +- stage: geographic_expand + survey: Family Resources Survey 2024-25 + source: Microcosm structural operator (microcosm#931); the FRS spine's own households, no publisher input + grain: household + artifacts: [] + operations: + - kind: expand_geographic_clones + entity: household + clone_index_columns: + person: person_clone_index + benunit: benunit_clone_index + household: household_clone_index + ordinal_bound: 100 + id_remapping: copy-id table field geographic_support; clone k takes every entity id + k x the field's radix (10**9) + order: clone_major + weight_split: equal + weight_kind_out: design + source_columns: + - source_household_id + - source_year + - source_household_key + conservation: household_total_to_rounding + declared_factor: 1.0 + reason: Geographic clones split each household's design weight equally among its clones, so household mass is conserved. + outputs: + - person_clone_index + - benunit_clone_index + - household_clone_index + - source_household_id + - source_year + - source_household_key + nonnegative_outputs: + - person_clone_index + - benunit_clone_index + - household_clone_index + - source_household_id + - source_year + rewrites: [] + notes: 'Gives every FRS household K geographic clones before any other copy step (microcosm#931), so that each clone can later take an area of its own while together they stand for the survey household once. Clone 0 is the frame''s own rows and ids; clone k repeats every person, benefit unit and household with each id minted in the copy-id table''s geographic_support field at ordinal k (id_remapping), and every table carries its rows'' clone, 0 to K - 1, in its clone index column. Each clone carries an equal share of its household''s design weight, the same float in every clone, and the frame''s mass log gains one record whose reason names no clone count. Every household carries its source_household_id (its clone-0 id), source_year (the frame''s year) and source_household_key, so the stages after it count true sources in survey households. K is a parameter of the stage''s graph node, never of this declaration, so the declaration and its contract are the same at every K. The stage runs after frs_hmrc_spine_leaves and frs_annual_income_spells, which join the raw FRS tabs on raw person ids, and before frs_take_up, frs_person_draws, frs_household_draws and frs_brma, which run on every clone. Its evidence gives K, the copy-id field and its radix, the distinct source households rounded by the shared disclosure rules, and the weight split and kind; no row count, id or weight.' +- stage: geographic_identity + survey: Family Resources Survey 2024-25 + source: Microcosm structural operator (microcosm#931); the FRS vintage frs_release.json pins, no publisher input + grain: household + artifacts: + - role: frs_release + kind: public_aggregate_reference + resource: frs_release.json + format: json + runtime_sha256_required: true + operations: + - kind: key_households_by_clone_path + source: frs + vintage_resource: frs_release.json + outputs: + - geography_household_key + rewrites: [] + notes: 'Keys every household for the area assignment (microcosm#931): geography_household_key, as the dense line''s uk.full.identity writes it, from the identity source (source), the FRS vintage frs_release.json pins for the household''s source_year, its source_household_id and its clone path, read from the lineage columns geographic_expand writes, never from ids, weights or values: no branch for clone 0 and the geographic_support branch at ordinal k for clone k. So clone 0 keys as the dense line keys the same base household, and growing K never changes an existing household''s key. A source year the release pins no vintage for, and keys that repeat, are refused by count. The copy steps after it copy a household''s key with the household. Its evidence gives the distinct source households rounded by the shared disclosure rules, that the keys are unique, the source, the vintages and the columns it reads; no key is printed.' +- stage: frs_take_up + survey: Family Resources Survey 2024-25 + source: Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates. + grain: benunit + artifacts: [] + operations: + - kind: aggregate_person_to_benunit + method: any_positive + consumed_only: true + aggregates: + child_benefit_reported_anchor: child_benefit_reported + pension_credit_reported_anchor: pension_credit_reported + universal_credit_reported_anchor: universal_credit_reported + - kind: aggregate_person_to_benunit + method: any_uc_claimant_aged_18_under_pension_credit_age + consumed_only: true + aggregates: + uc_age_eligible: age + role_column: is_uc_claimant + - kind: assign_binary_with_anchored_residual + output: would_claim_child_benefit + draw: would_claim_child_benefit + rate_key: child_benefit + anchor: child_benefit_reported_anchor + seed: 0 + - kind: assign_binary_from_rate + output: child_benefit_opts_out + draw: child_benefit_opts_out + rate_key: child_benefit_opts_out_rate + seed: 0 + - kind: assign_binary_with_anchored_residual + output: would_claim_pc + draw: would_claim_pc + rate_key: pension_credit + anchor: pension_credit_reported_anchor + seed: 0 + - kind: assign_binary_with_anchored_residual + output: would_claim_uc + draw: would_claim_uc + rate_key: universal_credit + anchor: universal_credit_reported_anchor + population: uc_age_eligible + seed: 0 + - kind: assign_binary_from_rate + output: would_claim_tfc + draw: would_claim_tfc + rate_key: tax_free_childcare + seed: 0 + - kind: assign_binary_from_rate + output: would_claim_extended_childcare + draw: would_claim_extended_childcare + rate_key: extended_childcare + seed: 0 + - kind: assign_binary_from_rate + output: would_claim_universal_childcare + draw: would_claim_universal_childcare + rate_key: universal_childcare + nested_under: would_claim_extended_childcare + rate_derivation: the contract's universal_childcare rate is the rate among benefit units not claiming the extended hours; would_claim_universal_childcare = would_claim_extended_childcare OR u < rate, so every unit claiming the extended hours claims the universal hours + seed: 0 + - kind: assign_binary_from_rate + output: would_claim_targeted_childcare + draw: would_claim_targeted_childcare + rate_key: targeted_childcare + seed: 0 + - kind: assign_binary_from_banded_rates + output: would_claim_uc_childcare + draw: would_claim_uc_childcare + band_column: is_married + bands: + - upper_exclusive: 1 + rate_key: uc_childcare_single + - rate_key: uc_childcare_couple + seed: 0 + - kind: assign_clipped_normal + output: maximum_extended_childcare_hours_usage + draw: maximum_extended_childcare_hours_usage + distribution_key: maximum_extended_childcare_hours_usage + seed: 0 + - kind: assign_binary_from_rate + output: max_free_entitlement_hours_used + draw: max_free_entitlement_hours_used + rate_key: extended_childcare_age_0_usage + eligibility: persons of DfE age 0 in benefit units that claim the working-parent hours (would_claim_extended_childcare) + assignment_unit: person + rate_derivation: a child of DfE age 0 in a claiming unit uses the working-parent hours where u < its age's rate and gets max_free_entitlement_hours_used 0 otherwise; every other person keeps the engine's default of 30; one draw per person serves both ages' operations + seed: 0 + - kind: assign_binary_from_rate + output: max_free_entitlement_hours_used + draw: max_free_entitlement_hours_used + rate_key: extended_childcare_age_1_usage + eligibility: persons of DfE age 1 in benefit units that claim the working-parent hours (would_claim_extended_childcare) + assignment_unit: person + rate_derivation: a child of DfE age 1 in a claiming unit uses the working-parent hours where u < its age's rate and gets max_free_entitlement_hours_used 0 otherwise; every other person keeps the engine's default of 30; one draw per person serves both ages' operations + seed: 0 + outputs: + - would_claim_child_benefit + - child_benefit_opts_out + - would_claim_pc + - would_claim_uc + - would_claim_tfc + - would_claim_extended_childcare + - would_claim_universal_childcare + - would_claim_targeted_childcare + - would_claim_uc_childcare + - maximum_extended_childcare_hours_usage + - max_free_entitlement_hours_used + nonnegative_outputs: + - maximum_extended_childcare_hours_usage + - max_free_entitlement_hours_used + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. The Universal Credit draw's population is the benefit units with a UC claimant or partner (is_uc_claimant) aged 18 or over and under Pension Credit qualifying age (66 for everyone who can reach it in this release), the age and role conditions of policyengine-uk 2.177.7's is_uc_eligible; the engine lockstep test pins it and names the engine cases not mirrored (16- and 17-year-old claimants under the UC Regs 2013 reg 8 exceptions, mixed-age couples on the Pension Credit route, from 2.128.0 pension-age families moved onto Universal Credit when their legacy benefit closes, SI 2014/1230 reg 60A, and from 2.173.2 a unit whose working-age claimants all receive education with no exception, since frs_education sets is_in_full_time_education from the FRS). So the contract rate is a share of those units, an anchor outside them stays true, and any other unit is never drawn; a unit with no claimant or partner of that age keeps its reported Housing Benefit, which a claim it cannot make no longer zeroes (#882, microcosm#867, microcosm#1095). The universal childcare flag is nested under the working-parent one. A unit that claims the extended hours claims the universal hours, and any other unit claims them at the contract's universal_childcare rate, the rate conditional on not claiming the extended hours; the extended draw is unchanged. childcare_copy_take_up later re-draws the childcare flags and hours of the support and CGT copies on their own benefit units. A child of DfE age 0 or 1 (its age on 31 December of the fit's model year, the age DfE counts by) in a unit that claims the working-parent hours uses them at its age's contract rate, extended_childcare_age_0_usage or extended_childcare_age_1_usage (DfE registers far fewer of the youngest eligible children than of those aged 2 to 4, and fewer at 9 to 11 months than at age 1); a child that does not gets max_free_entitlement_hours_used 0, so the engine funds it no hours, and every other person keeps the engine's default of 30. +- stage: frs_person_draws + survey: Family Resources Survey 2024-25 + source: Sourced UK take-up contract rates and identity-keyed deterministic draws. + grain: person + artifacts: + - role: stochastic_contract + resource: take_up_contract.json + kind: public_aggregate_reference + format: json + operations: + - kind: assign_binary_from_rate + output: would_claim_marriage_allowance + draw: would_claim_marriage_allowance + rate_key: marriage_allowance + seed: 0 + - kind: assign_binary_from_banded_rates + output: would_claim_scp + draw: would_claim_scp + band_column: age + bands: + - upper_exclusive: 6 + rate_key: scp_under_6 + - rate_key: scp_6_plus + seed: 0 + - kind: assign_uniform_draw + output: attends_private_school_random_draw + seed: 0 + - kind: assign_period_constant + output: tax_free_childcare_spend_routed_share + contract_key: tax_free_childcare_spend_routed_share + period_source: build_year + dtype: float64 + - kind: assign_binary_from_rate + output: is_apprentice + draw: is_apprentice + rate_key: apprentice_stock_19_to_24 + assignment_unit: person + eligibility: 'employees (employment_status FT_EMPLOYED or PT_EMPLOYED, from frs_employment) aged 21 to 24 who are not in full-time education (is_in_full_time_education, from frs_education; the FRS codes no apprentice as in full-time education): SSCBA 1992 s.9A already zero-rates every earner under 21 and s.9B ends the apprentice zero rate at 25, so no other age changes employer NICs, and leaving them False keeps is_in_non_advanced_education, minimum_wage, Care to Learn and the 16 to 19 Bursary Fund unchanged; the flagged keep their age-related minimum_wage too, because policyengine-uk 2.162.8 (policyengine-uk#2683) gives the apprentice rate only under 19 or with is_in_first_year_of_apprenticeship, which the build leaves at its default (False), and the stage refuses an engine without that split or the s.9B threshold' + rate_derivation: 'the contract''s apprentice_stock_19_to_24 is DfE''s England apprentices in learning at the start of the academic year (participation less starts) per head aged 19 to 24, a point-in-time count taken as uniform across those ages and UK-wide; the target is r times the weighted population aged 21 to 24 at the frame''s household weights, and eligible employees are flagged in the order of their identity-keyed uniform until their weight is nearest it (uc_take_up''s rule), or all of them when they weigh no more than it; the receipt records the target, the flagged weight, the support bands and the relief the flags imply at the installed engine''s Class 1 schedule, and uk_stage_frs_person_draws_apprentices gates them, the relief at or below HMRC''s NIC - NS1' + seed: 0 + outputs: + - would_claim_marriage_allowance + - would_claim_scp + - attends_private_school_random_draw + - tax_free_childcare_spend_routed_share + - is_apprentice + nonnegative_outputs: + - attends_private_school_random_draw + - tax_free_childcare_spend_routed_share + notes: 'Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. is_apprentice has no FRS source (no apprenticeship variable in the 2022-23 to 2024-25 adult tables; TRAIN2, NI3TRAIN and TRAINEE code government employment schemes) and is drawn to DfE''s point-in-time rate for employees aged 21 to 24 only.' +- stage: frs_household_draws + survey: Family Resources Survey 2024-25 + source: Sourced UK stochastic contract rates and identity-keyed deterministic draws. + grain: household + artifacts: [] + operations: + - kind: assign_binary_from_rate + output: household_owns_tv + draw: household_owns_tv + rate_key: tv_ownership_rate + seed: 0 + - kind: assign_binary_from_rate + output: would_evade_tv_licence_fee + draw: would_evade_tv_licence_fee + rate_key: tv_licence_evasion_rate + seed: 0 + - kind: assign_binary_from_rate + output: main_residential_property_purchased_is_first_home + draw: main_residential_property_purchased_is_first_home + rate_key: first_time_buyer_rate + seed: 0 + - kind: assign_binary_from_rate + output: property_purchased + draw: property_purchased + rate_key: property_purchase_rate + seed: 0 + outputs: + - household_owns_tv + - would_evade_tv_licence_fee + - main_residential_property_purchased_is_first_home + - property_purchased + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share. +- stage: frs_brma + survey: Family Resources Survey 2024-25 + source: Census private-rented households by region, BRMA and bedrooms (ONS and NISRA 2021, NRS 2022) and policyengine-uk LHA_category predictor. + grain: household + artifacts: + - role: count_resource + resource: brma_private_rented_households.json + kind: public_aggregated_counts + format: json + operations: + - kind: materialize_rules_engine_predictors + predictors: + - LHA_category + consumed_only: true + - kind: sample_categorical_from_count_table + output: brma + count_resource: brma_private_rented_households.json + cell_columns: + - region + - LHA_category + assignment_grain: benunit + collapse_to: household + collapse_method: uniform_member_pick + seed: 0 + outputs: + - brma + notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick. - stage: spi_support_channel survey: Family Resources Survey 2024-25 - source: Synthetic SPI support channel drawn from the raw FRS spine as a keyed Bernoulli share, each household on its own identity-keyed uniform. + source: Synthetic SPI support channel drawn from every geographic clone of the raw FRS spine's households as a keyed Bernoulli share, each household on its own identity-keyed uniform. grain: household artifacts: [] operations: @@ -1555,9 +1614,6 @@ stages: - household_support_channel - household_support_clone_index - household_source_id - - source_household_id - - source_year - - source_household_key nonnegative_outputs: - person_support_clone_index - person_source_id @@ -1565,9 +1621,7 @@ stages: - benunit_source_id - household_support_clone_index - household_source_id - - source_household_id - - source_year - notes: "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's draw depends on its own id alone. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." + notes: "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's uniform depends on its own id and the stage's seed and salt alone, and whether it is copied also on that probability. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." - stage: spi_income_band_donors survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5 source: Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane). @@ -1606,11 +1660,11 @@ stages: minimum_donors_per_band: 120 maximum_donor_weight: 250.0 count_rule: max(minimum_donors_per_band, ceil(published band taxpayers / maximum_donor_weight)) - weight_rule: published band taxpayers / (geographic clone count x band donor count), equal within a band + weight_rule: published band taxpayers / (geographic clone count x one clone's band donor count), equal within a band expected_band_count: 4 candidate_population: raw FRS base channel claimants and partners (is_uc_claimant) aged 16 and over candidate_order: person_id ascending breaks equal draw keys - draw: 'weighted_without_replacement_identity_keyed within each geographic clone (one before uk.full.expand): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first' + draw: 'weighted_without_replacement_identity_keyed within each geographic clone (a spine without household_clone_index is one clone): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first' propensity: SPI 2022-23 FACT-weighted band share by region, sex and age band tape_preparation: 'as hmrc_spi_income_spine prepares the tape: its stage-1 seed and draw_spi_donor_ages_by_population' tape_preparation_seed: 42 @@ -1650,7 +1704,7 @@ stages: - household_source_id nonnegative_outputs: - spi_income_band_donor_lower_bound - notes: Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, with the seats counted in distinct source households, each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves). + notes: Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, and every clone takes the same seats, sized against the eligible households counted in distinct source households. Each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves). - stage: hmrc_spi_income_spine survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods @@ -3331,7 +3385,7 @@ stages: stratum_rule: within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected seed: 0 salt: high_value_home_support:value_tie - copy_rule: every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight + copy_rule: every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight id_remapping: copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12) flag_column: household_is_high_value_support_copy copies_column: high_value_support_copies @@ -5597,7 +5651,7 @@ stages: rewrites: - frs_benunit_capital - would_claim_uc - notes: Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donors widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (María's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base. + notes: Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donor survey records (a unit's geographic clones count once, every other copy as a record of its own) widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (María's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base. - stage: uc_limited_capability survey: Family Resources Survey 2024-25 (BENEFITS.UCLCW, DWP's limited capability status on a linked Universal Credit claim) and policyengine-uk's limited capability proxy, evaluated by the pinned engine source: frs_uc_lcw_status (frs_spine) on the UC row's person, and the locked engine's uc_limited_capability_for_WRA, uc_limited_capability_for_work, is_uc_assessed_claimant and is_carer_for_benefits on the frame as it stands @@ -6020,7 +6074,7 @@ stages: seed: 0 salt: cgt_support_split:wealth_tie maximum_copy_weight: 60.0 - copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make + copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make id_remapping: copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused flag_column: household_is_cgt_support_copy copies_column: cgt_support_copies @@ -6035,7 +6089,7 @@ stages: - cgt_support_copies - cgt_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool''s selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk''s tie-break, a uniform keyed on each household''s own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' + notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool''s selection recompute can then count that household as a family the stage never made, which the bracketed family test does not report, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk''s tie-break, a uniform keyed on each household''s own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' - stage: cgt_incidence_clone survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 @@ -6587,7 +6641,7 @@ stages: candidates: a bonus (frs_bonus_amount, or an SPI-channel adult's drawn bonus) > 0 and employment_income at least the anchor's annual upper earnings limit, after the salary conversion move: the bonus, at most move_pay_share_cap of employment_income, to pension_contributions_via_bonus_sacrifice; employment_income unchanged, since a sacrificed bonus is never received move_pay_share_cap: 0.5 - spi_bonus_draw: an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell, by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below + spi_bonus_draw: an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell and the same geographic clone (household_clone_index; a spine without it is one clone), by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles over every clone at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below, and the lowest the one above spi_cell_quantiles: - 0.1 - 0.2 @@ -6623,7 +6677,7 @@ stages: - pension_contributions_via_salary_sacrifice - employee_pension_contributions - employment_income - notes: 'The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household''s members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin''s answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors'' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR''s 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (María''s ruling of 2026-10-24 on Vahid''s review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult''s FRS bonus is its twin''s, beside the tape''s pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell instead.' + notes: 'The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household''s members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin''s answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors'' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR''s 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (María''s ruling of 2026-10-24 on Vahid''s review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult''s FRS bonus is its twin''s, beside the tape''s pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell, in its own geographic clone, instead.' - stage: carers_allowance_take_up survey: Family Resources Survey 2024-25 source: 'Independent Review of Carer''s Allowance Overpayments (July 2025), chapter 7, and DWP benefit statistics, through the UK take-up contract''s carers_allowance_newly_entitled rate, applied to PolicyEngine-UK''s Carer''s Allowance earnings test after the SPI income chain.' @@ -6803,10 +6857,10 @@ stages: - would_claim_targeted_childcare - maximum_extended_childcare_hours_usage - max_free_entitlement_hours_used - notes: Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units before any copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id. + notes: Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units, each geographic clone's on its own ids (microcosm#931), before any other copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units, every geographic clone's included, keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id. - stage: private_school_attendance survey: policyengine-uk private-school attendance imputation, stored at the spine's weights - source: policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine + source: policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor) over every person in the frame, from the pinned engine's values and parameters grain: person artifacts: - role: frs_release @@ -6827,7 +6881,7 @@ stages: gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: - attends_private_school - notes: 'The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine''s attendance once, at the calibration year and the spine''s design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula''s last step over every person in the frame, from the engine''s own values at the calibration year, each decided by a household''s own records, and its own parameters: household market income plus benefits less the members'' income tax and National Insurance, each household''s weight once per member, microdf''s weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine before its geographic clones exist), the stage stores the engine''s flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine''s flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.' + notes: 'The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage computes the formula''s attendance once, over the whole frame at the calibration year and the spine''s design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula''s last step over every person in the frame, from the engine''s own values at the calibration year, each decided by a household''s own records, and its own parameters: household market income plus benefits less the members'' income tax and National Insurance, each household''s weight once per member, microdf''s weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine of one geographic clone), the stage stores the engine''s flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine''s flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.' - stage: isa_subscriptions survey: HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8 source: HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain). @@ -6962,7 +7016,7 @@ stages: - cash_isa_subscriptions - stocks_and_shares_isa_subscriptions - lifetime_isa_payments - notes: 'Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household''s was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7''s income bands, Table 9.8''s age and sex and Table 9.9''s regions and selected by keyed walks, the both class first. Amounts follow Table 9.7''s subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person''s amount by their balance shares, shifted so the class''s stocks-and-shares total is Table 9.4''s 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. People grow to 2024-25 by Table 9.4''s account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4''s 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7''s last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (María''s rulings of 29 October 2026): Table 9.7''s cash bands sit above Table 9.4, and growing that excess by Table 9.4''s ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4''s 2024-25 Lifetime ISA amount. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.' + notes: 'Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household''s was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7''s income bands, Table 9.8''s age and sex and Table 9.9''s regions and selected by keyed walks, the both class first. Amounts follow Table 9.7''s subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person''s amount by their balance shares, shifted so the class''s stocks-and-shares total is Table 9.4''s 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. Amounts go in the order of a key that blends each subscriber''s weighted rank on the class''s balance (cash, stocks and shares, or the two together) with a keyed uniform at balance_rank_weight; tied balances share one weighted mid-rank, so the keyed uniform alone orders them. People grow to 2024-25 by Table 9.4''s account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4''s 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7''s last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (María''s rulings of 29 October 2026): Table 9.7''s cash bands sit above Table 9.4, and growing that excess by Table 9.4''s ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4''s 2024-25 Lifetime ISA amount and ordered by the same blend, on the Lifetime ISA balance over every holder aged 18 to 47, with tied balances again sharing one mid-rank; the Lifetime ISA operation declares no rank weight and reuses the amounts operation''s balance_rank_weight. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.' - stage: pip_points survey: DWP's Pathways to Work evidence pack, chapter 2, Tables 2.9 and 2.10 (PIP points by age group, England and Wales, end of January 2025), on the spine's stored PIP awards source: pip_dl_category and pip_m_category (frs_disability, and hmrc_spi_income_spine for the SPI rows) and the stored age; dwp_pip_points_facts.json, vendored from the pinned feed diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py index 26a98b9eaa..40c07e958e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py @@ -22,6 +22,14 @@ validate_assignment_spec, ) from microcosm.build.atomic_support_conditioning import extend_atomic_support + +# The household key the assignment draws on. The clone-first spine's +# geographic_identity stage writes it too, so the name lives in the +# geographic_clones leaf, which that stage hashes; re-exported here, so a +# kernel that reads it here is held to the leaf (microcosm#931). +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN as IDENTITY_COLUMN, +) from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE from microcosm.graph.canonical import canonical_json @@ -31,7 +39,6 @@ "uk_ni_data_zone_2021", ) SOURCES = {system: system + "_support" for system in SYSTEMS} -IDENTITY_COLUMN = "geography_household_key" #: Assignment outputs the shared operators write on the household table. ASSIGNMENT_COLUMNS = ("atomic_area_code", "atomic_area_system", "atomic_area_basis") #: Nation-native aliases of derived layers; NA outside their own nation and diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index ae5ca0d14e..6f92b485aa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -152,13 +152,14 @@ CGT_SUPPORT_CORRECTION_CANDIDATES = 64 #: The one exception to "every copy carries weight / copies" that the #: ``copy_rule`` of both support splits declares, built on the cap so the -#: declarations restate it (microcosm#931). The CGT split's adds +#: declarations restate it, with the order the candidates are tried in +#: (:func:`_exact_support_weights`; microcosm#931). The CGT split's adds #: :data:`CGT_SUPPORT_ONE_COPY_CHECK` after it. CGT_SUPPORT_EXACT_TOTAL_EXCEPTION = ( f"except where none of the {CGT_SUPPORT_CORRECTION_CANDIDATES} heaviest " - "households left whole can carry the exact-total correction, when the " - "shared fallback can put it on a root or copy, off weight / copies by " - "about one ulp of the total weight" + "households left whole, tried heaviest first, tied weights in row order, " + "can carry the exact-total correction, when the shared fallback can put it " + "on a root or copy, off weight / copies by about one ulp of the total weight" ) #: The clause the CGT split's ``copy_rule`` adds after that exception, built #: on the cap too. Of those candidates, a selected household left at one copy diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/childcare_copy_take_up.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/childcare_copy_take_up.py index 48cc10a4ad..741248a783 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/childcare_copy_take_up.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/childcare_copy_take_up.py @@ -1,7 +1,8 @@ """Re-draw the childcare take-up of support and CGT copies on their own units. ``frs_take_up`` draws the childcare flags and the extended-hours usage keyed on -each FRS benefit unit before any copy exists, so every copy the later stages +each FRS benefit unit, each geographic clone's on its own ids +(microcosm#931), before any other copy exists, so every copy the later stages make (the SPI support channel and its income-band donors, the high-value home support, the CGT support split, the CGT incidence clone and the residential arms) inherits its host's draws. A small cell of hosts then fixes the take-up @@ -22,8 +23,10 @@ id in the frame, so a copy's id is unique and differs from every native id: the copy's streams are independent of its host's, and a rebuild of the same frame draws the same values. Native benefit units keep their ``frs_take_up`` -draws untouched. Anchored flags (Child Benefit, Pension Credit, Universal -Credit) and ``would_claim_uc_childcare`` are left as they are. +draws untouched, every geographic clone's included, since a geographic +clone is not a copy here (``without_geographic_branch``). Anchored flags +(Child Benefit, Pension Credit, Universal Credit) and +``would_claim_uc_childcare`` are left as they are. A copy is a benefit unit whose household carries a non-empty clone path (``uk_household_clone_paths``: the SPI support channel, the CGT support split, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py index 4c31b6c1fe..20bbb57881 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py @@ -18,8 +18,9 @@ reads one FRS release today, so nothing mints it; raw FRS ids (``household_id`` the sernum, ``benunit_id`` sernum x 100 + b, ``person_id`` sernum x 1000 + p) must stay below its radix. -- ``geographic_support``: the geographic clones the clone-first spine will - mint at its start (K at most 100). Nothing mints it yet. +- ``geographic_support``: the geographic clones the spine's + ``geographic_expand`` stage mints before any other copy layer runs (K at + most 100). - ``spi_support_channel``: the SPI support copy (ordinal 1, stage 16) and the income band donor (ordinal 2, stage 17). - ``high_value_support``: the high-value home copies (stage 26). diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py index 24a8985627..a5afc180b5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py @@ -533,7 +533,9 @@ def uk_weight_summary( def uk_support_family_geographic_clone_column() -> str: - """The household geographic clone-index column ``uk.full.expand`` writes.""" + """The household geographic clone-index column: the spine's + ``geographic_expand`` stage writes it (microcosm#931), and ``uk.full.expand`` + on a spine built before that stage.""" from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 9c3f0382d1..8ae1cb27dd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -23,7 +23,9 @@ ) from microcosm.build.uk_runtime.council_tax_constants import ( UK_COUNCIL_TAX_BANDS, + UK_COUNCIL_TAX_BANDS_BY_REGION, UK_COUNCIL_TAX_PLACEHOLDER_BAND, + UK_NATION_BY_REGION, ) from microcosm.build.uk_runtime.national_frame import ( uk_national_frame, @@ -44,9 +46,12 @@ from microcosm.frame import Frame, WeightKind __all__ = [ + "FRS_AREA_CONDITIONING_COLUMNS", "FRS_CARE_HELPER_BIT", "FRS_CARE_LINE_HOURS", "FRS_CARE_RECIPIENTS_COLUMN", + "FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN", + "FRS_IMD_DECILE_COLUMN", "FRS_SPINE_TABLES", "FRS_UC_CARER_ELEMENT_COLUMN", "OUTPUT_COLUMNS", @@ -157,6 +162,41 @@ code: band for code, band in enumerate(UK_COUNCIL_TAX_BANDS, start=1) } +#: The two household carriers the clone-first spine's area assignment +#: conditions on (microcosm#931): each household's IMD decile within its own +#: nation, and whether its council_tax_band is a band the FRS gives it that +#: its region uses. The root writes them typed (an int64 decile, 0 where none +#: is observed, since the root refuses NaN; a bool flag); neither is an engine +#: input, so they leave at the release boundary +#: (``national_frame.UK_RELEASE_EXPORT_DROPPED_COLUMNS``), and they stay out +#: of every receipt and of the spine sidecar's nonzero shares. The decile's +#: name is spelled here, not imported from ``atomic_area_support``, whose +#: conditioned supports carry the areas' deciles under the same name, so an +#: edit to that module does not re-key the root; a test holds the spellings +#: equal. +FRS_IMD_DECILE_COLUMN = "imd_decile" +FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN = "council_tax_band_observed" +FRS_AREA_CONDITIONING_COLUMNS = ( + FRS_IMD_DECILE_COLUMN, + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, +) +#: Each nation's FRS household IMD decile variable (HOUSEHOL IMDE, IMDW, IMDS +#: and IMDN), by the nation names of ``council_tax_constants``: the +#: deprivation tenth of the address within that nation's own index, 1 the +#: most deprived. A household reads its own nation's variable only. +FRS_IMD_DECILE_COLUMN_BY_NATION: Mapping[str, str] = MappingProxyType( + { + "ENGLAND": "imde", + "WALES": "imdw", + "SCOTLAND": "imds", + "NORTHERN_IRELAND": "imdn", + } +) +#: The decile codes; anything else, a blank included, is no observed decile. +FRS_IMD_DECILES = tuple(range(1, 11)) +#: The decile a household with no observed decile carries. +FRS_IMD_DECILE_UNOBSERVED = 0 + MARITAL_MAP = { 1: "MARRIED", 2: "SINGLE", @@ -558,11 +598,13 @@ def benefits_by_recipient( "liable_for_share_of_household_rent", "household_id", "region", + FRS_IMD_DECILE_COLUMN, "tenure_type", "accommodation_type", "num_bedrooms", "council_tax_reported", "council_tax_band", + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, "council_tax_rebate", "council_tax_single_adult_raw", "water_and_sewerage_charges", @@ -1853,6 +1895,7 @@ def _add_household_columns( pe_household["region"] = _map_dictionary_codes( household, "gvtregno", REGION_MAP, blank="UNKNOWN" ) + pe_household[FRS_IMD_DECILE_COLUMN] = frs_imd_decile(household) pe_household["tenure_type"] = _map_dictionary_codes( household, "ptentyp2", TENURE_MAP, blank="RENT_PRIVATELY" ) @@ -1864,6 +1907,9 @@ def _add_household_columns( pe_household["num_bedrooms"] = _number(household, "bedroom6") pe_household["council_tax_reported"] = _positive(household, "ctannual") pe_household["council_tax_band"] = frs_council_tax_band(household) + pe_household[FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN] = frs_council_tax_band_observed( + household + ) pe_household["council_tax_rebate"] = ( _positive(household, "ctrebamt") * WEEKS_IN_YEAR ) @@ -1963,6 +2009,70 @@ def frs_council_tax_band(household: pd.DataFrame) -> pd.Series: return band.mask(region == "NORTHERN_IRELAND", UK_COUNCIL_TAX_PLACEHOLDER_BAND) +def frs_council_tax_band_observed(household: pd.DataFrame) -> pd.Series: + """Whether each household's ``council_tax_band`` is a band the FRS gives it + that its region uses. + + True where ``ctband`` codes a band, as ``frs_council_tax_band`` maps the + codes 1-9, that the household's region uses + (``UK_COUNCIL_TAX_BANDS_BY_REGION``: A-H in England and Scotland, A-I in + Wales). False in Northern Ireland, which pays domestic rates, for any + other code, which takes the placeholder band, for a band the region does + not use (band I outside Wales), and for a household of no known region + (microcosm#931). The band itself is unchanged; the flag tells the + survey's band from the placeholder and from a band its region does not + hold. + """ + + # The band each code maps to, by the band's own mapping; an unmapped code + # takes the empty string, which no region uses. + bands = _map_codes(household, "ctband", COUNCIL_TAX_BAND_MAP, "") + region = _map_dictionary_codes(household, "gvtregno", REGION_MAP, blank="UNKNOWN") + region_bands = region.map(lambda name: UK_COUNCIL_TAX_BANDS_BY_REGION.get(name, ())) + in_region = np.fromiter( + ( + band in allowed + for band, allowed in zip(bands.tolist(), region_bands.tolist(), strict=True) + ), + dtype=bool, + count=len(household), + ) + observed = in_region & (region != "NORTHERN_IRELAND").to_numpy(dtype=bool) + return pd.Series(observed, index=household.index, dtype=bool) + + +def frs_imd_decile(household: pd.DataFrame) -> pd.Series: + """Each household's IMD decile within its own nation: 1-10, else 0. + + The FRS gives the decile of each address on its own nation's index, one + variable per nation (``FRS_IMD_DECILE_COLUMN_BY_NATION``); a household + reads its own nation's variable, its nation coming from its region. A + code outside 1-10, a blank, and a household of no known region carry 0, + no observed decile, since the root refuses NaN (microcosm#931). A tab + without any of the four variables refuses, naming them, rather than read + an absent variable as zeros and so give nobody a decile. + """ + + missing = [ + column.upper() + for column in FRS_IMD_DECILE_COLUMN_BY_NATION.values() + if column not in household.columns + ] + if missing: + raise KeyError( + f"The FRS household tab has no {missing} column(s), so the IMD " + "decile cannot be set." + ) + region = _map_dictionary_codes(household, "gvtregno", REGION_MAP, blank="UNKNOWN") + nation = region.map(UK_NATION_BY_REGION) + decile = np.full(len(household), FRS_IMD_DECILE_UNOBSERVED, dtype=np.int64) + for name, column in FRS_IMD_DECILE_COLUMN_BY_NATION.items(): + codes = _raw_number(household, column) + rows = ((nation == name) & codes.isin(FRS_IMD_DECILES)).to_numpy(dtype=bool) + decile[rows] = codes.to_numpy(dtype=np.float64)[rows].astype(np.int64) + return pd.Series(decile, index=household.index, dtype="int64") + + def _map_codes( frame: pd.DataFrame, column: str, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py index b62cbd78f7..70cb2da3a6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py @@ -832,8 +832,11 @@ def uk_take_up_signal_receipt( flags.append(row) continue if entity not in sources: - # Before the SPI support channel each household is its own source, - # so its id serves until ``source_household_id`` exists. + # On the clone-first spine geographic_expand writes + # source_household_id before this stage runs, so the shared reader + # counts a household's geographic clones once; a frame without the + # column or copy lineage (unit fixtures) counts each household as + # its own source (microcosm#931). sources[entity] = record_source_households( frame, entity, stage="The take-up signal receipt", before_copies=True ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py index 5917eb0f4a..291ca86fcc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py @@ -74,13 +74,18 @@ def uk_spine_transferred_only_code_hash() -> str: The assembled gate's own key, on which every later spine stage's chains, therefore carries none of them (the re-review of microcosm#1420, E; microcosm#931 ruling 6). An edit to one still re-keys the assembled gate - wherever a stage before it hashes the module: stages at or before - ``frs_brma``, the assembled phase's last, hash ``stochastic_assignment`` - and ``take_up_contract``, so an edit to either re-keys every spine node - after them, the assembled gate included. No stage before that gate hashes - the other fifteen, so an edit to one of those re-keys the stages that - hash it, those after them and the transferred gate, never the assembled - gate. Hashed once per process, as the assembled phase's own code is. + wherever a stage before it hashes the module. Stages at or before + ``frs_brma``, the assembled phase's last, hash six of them: + ``stochastic_assignment`` and ``take_up_contract``; ``frs_annual_income``, + the module of ``frs_annual_income_spells``; and the vendored-fact reader, + the chronicle feed and its schema epochs, which ``frs_hmrc_spine_leaves`` + binds. Those two stages run before ``frs_brma`` on the clone-first spine + (microcosm#931). So an edit to any of the six re-keys every spine node + from the first stage that hashes it onwards, the assembled gate + included. No stage before that gate hashes the other eleven, so an edit + to one of those re-keys the stages that hash it, those after them and + the transferred gate, never the assembled gate. Hashed once per process, + as the assembled phase's own code is. """ from .. import chronicle_epoch, stochastic_assignment from .. import ledger_targets as shared_ledger_targets diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py index 0cac7cace6..ce49f61514 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py @@ -1,4 +1,4 @@ -"""The UK geographic clone index columns' names, and a frame's clone count. +"""The UK geographic clones' column names, and a frame's clone count. Each geographic clone of a household carries its index, 0 to ``K - 1``, on the household, its benefit units and its people, under a per-entity name @@ -6,18 +6,22 @@ schema's one shared name (:data:`ARTIFACT_CLONE_INDEX_COLUMN`). The UK copy lineage reads the household's index as its ``geographic_support`` branch (``household_lineage.UK_COPY_LINEAGE``), and :func:`geographic_clone_count` -reads ``K`` from it. +reads ``K`` from it. The spine's ``geographic_expand`` stage writes the +indices and every household's source columns +(:data:`UK_GEOGRAPHIC_SOURCE_COLUMNS`), and its ``geographic_identity`` +stage each clone's geography key (:data:`GEOGRAPHY_HOUSEHOLD_KEY_COLUMN`), +which the dense line's ``atomic_area_support`` names ``IDENTITY_COLUMN``. -The two names lived in ``rowwise_dataset`` and the count in ``cgt_support``, -which re-export them. This module holds the three alone and imports nothing -first-party, as ``annuity_trust_column`` holds its column's name. The stages -that count by lineage read the household's name through ``household_lineage``, -and their nodes hash this module with the counting modules -(``graph_kernels._COUNTING_MODULES``); the CGT support split, which reads -``K`` through the count, hashes it too. The full build's population and export -kernels, which read the names, hash it as well. So an edit here re-keys all -of them, without binding the dense line's ``rowwise_dataset`` into the spine -(microcosm#931). +The two index names lived in ``rowwise_dataset`` and the count in +``cgt_support``, which re-export them. This module holds the names and the +count alone and imports nothing first-party, as ``annuity_trust_column`` +holds its column's name. The stages that count by lineage read the +household's name through ``household_lineage``, and their nodes hash this +module with the counting modules (``graph_kernels._COUNTING_MODULES``); the +CGT support split, which reads ``K`` through the count, hashes it too. The +full build's population and export kernels, which read the names, hash it as +well. So an edit here re-keys all of them, without binding the dense line's +``rowwise_dataset`` or ``atomic_area_support`` into the spine (microcosm#931). """ from __future__ import annotations @@ -27,6 +31,10 @@ __all__ = [ "ARTIFACT_CLONE_INDEX_COLUMN", + "GEOGRAPHY_HOUSEHOLD_KEY_COLUMN", + "SOURCE_HOUSEHOLD_KEY_COLUMN", + "SOURCE_YEAR_COLUMN", + "UK_GEOGRAPHIC_SOURCE_COLUMNS", "geographic_clone_count", "ladder_clone_index_column", ] @@ -34,6 +42,27 @@ #: Clone-index column name in the written artifact (one name per table — #: the legacy single-year schema, unchanged by the Frame carrier). ARTIFACT_CLONE_INDEX_COLUMN = "clone_index" +#: The household's survey year, which ``geographic_expand`` writes on every +#: household. +SOURCE_YEAR_COLUMN = "source_year" +#: ``f"{source_year}:{source_household_id}"``, which ``geographic_expand`` +#: writes on every household. +SOURCE_HOUSEHOLD_KEY_COLUMN = "source_household_key" +#: The households' source columns ``geographic_expand`` writes on every +#: household: the source id (each household's clone-0 id), its year and its +#: key. The source id is ``microcosm.frame.concepts.SOURCE_HOUSEHOLD_ID_COLUMN``, +#: spelled here so that this module imports nothing first-party; a test holds +#: the two spellings equal. +UK_GEOGRAPHIC_SOURCE_COLUMNS = ( + "source_household_id", + SOURCE_YEAR_COLUMN, + SOURCE_HOUSEHOLD_KEY_COLUMN, +) +#: Each clone's geography key, which ``geographic_identity`` writes on every +#: household of the spine and the dense line's ``uk.full.identity`` on every +#: household of its pool (``atomic_area_support.IDENTITY_COLUMN`` is this +#: name). +GEOGRAPHY_HOUSEHOLD_KEY_COLUMN = "geography_household_key" def ladder_clone_index_column(entity: str) -> str: @@ -50,13 +79,13 @@ def ladder_clone_index_column(entity: str) -> str: def geographic_clone_count(household: pd.DataFrame) -> int: """The frame's geographic clone count ``K``: one more than its largest index. - The clone-first spine (microcosm#931) gives every household the index of - its geographic clone in ``household_clone_index``, 0 to ``K - 1``; a - household table without that column, as on every national spine today, - is one clone. Refuses a non-integer index column, a household without an - index, a negative index and indices that are not exactly 0 to ``K - 1`` - (an index below the largest that no household carries). Each refusal - gives a count or the column's dtype, never an id or a value. + The clone-first spine's ``geographic_expand`` stage (microcosm#931) gives + every household the index of its geographic clone in + ``household_clone_index``, 0 to ``K - 1``; a household table without that + column is one clone. Refuses a non-integer index column, a household + without an index, a negative index and indices that are not exactly 0 to + ``K - 1`` (an index below the largest that no household carries). Each + refusal gives a count or the column's dtype, never an id or a value. """ column = ladder_clone_index_column("household") diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py index 55c3d38414..867123b5f5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py @@ -1,7 +1,7 @@ """The UK spine's geographic clones: K copies of every FRS household (microcosm#931). The clone-first spine gives every FRS household ``K`` geographic clones before -any copy step runs. Each clone carries an equal share of its household's +any other copy step runs. Each clone carries an equal share of its household's design weight, so a later stage can give each clone an area of its own while together they stand for the survey household once. This module is that step, G1 of the clone-first plan: :class:`UKGeographicExpandStageTransform` and the @@ -29,16 +29,18 @@ - It refuses a frame that already carries a clone index, a source column or any of the UK copy lineage's columns, since the step runs once, before any - copy. It also refuses weights on any entity but the household: the core - splits every weighted entity, and the UK weights households only. And it - refuses id and membership columns that are not int64, the dtype the + other copy step. It also refuses weights on any entity but the household: + the core splits every weighted entity, and the UK weights households only. + And it refuses id and membership columns that are not int64, the dtype the copy-id table mints, at every clone count, so every clone count takes the same frames. - Before the core runs, it writes each household's ``source_year`` (the - frame's time period, as the SPI support channel reads it) and - ``source_household_key`` (``f"{source_year}:{source_household_id}"``, the - values that stage writes, as Python strings in an object column) on the - frame's own rows, so every clone carries its household's. + frame's time period) and ``source_household_key`` + (``f"{source_year}:{source_household_id}"``, as Python strings in an object + column) on the frame's own rows, so every clone carries its household's: + the values the SPI support channel wrote before this stage took the + columns over, as it still does on a household table without the clone + index. - It validates the clones as a UK national frame. Every refusal gives counts, entities, columns, dtypes and declared constants @@ -47,7 +49,12 @@ rounded through ``receipt_counts``, and the weight split and kind; no row count, id or weight. -No stage runs this step yet, so it moves no output and no node key. +The spine's ``geographic_expand`` stage runs this step, after +``frs_annual_income_spells`` and before ``geographic_identity``. Its +declared operation, ``expand_geographic_clones``, must be the reviewed one +(:func:`geographic_expand_operation_parameters`), which names no clone +count: ``K`` is a parameter of the stage's graph node, so it enters the keys +of that node and every later one, never the stage's contract. """ from __future__ import annotations @@ -64,7 +71,13 @@ ) from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime import copy_id_table -from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations +from microcosm.build.uk_runtime.geographic_clones import ( + SOURCE_HOUSEHOLD_KEY_COLUMN, + SOURCE_YEAR_COLUMN, + UK_GEOGRAPHIC_SOURCE_COLUMNS, + ladder_clone_index_column, +) from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE from microcosm.build.uk_runtime.national_frame import ( UK_NATIONAL_SCHEMA, @@ -82,7 +95,11 @@ __all__ = [ "GEOGRAPHIC_CLONE_LIMIT", "GEOGRAPHIC_COPY_ID_FIELD", + "GEOGRAPHIC_EXPAND_CONSERVATION", + "GEOGRAPHIC_EXPAND_ID_REMAPPING", "GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON", + "GEOGRAPHIC_EXPAND_OPERATION_KIND", + "GEOGRAPHIC_EXPAND_ORDER", "GEOGRAPHIC_EXPAND_STAGE_NAME", "GEOGRAPHIC_EXPAND_WEIGHT_SPLIT", "SOURCE_HOUSEHOLD_KEY_COLUMN", @@ -93,6 +110,7 @@ "UKGeographicExpandResult", "UKGeographicExpandStageTransform", "expand_uk_geographic_clones", + "geographic_expand_operation_parameters", "mint_geographic_clone_ids", "require_room_for_geographic_clones", ] @@ -103,17 +121,10 @@ #: The most clones the field numbers: clone ordinals run below it. Exported #: so the spine and its graph can read the limit from here. GEOGRAPHIC_CLONE_LIMIT = copy_id_table.ordinal_bound(GEOGRAPHIC_COPY_ID_FIELD) -#: The household's survey year, which the step writes on every household. -SOURCE_YEAR_COLUMN = "source_year" -#: ``f"{source_year}:{source_household_id}"``, which the step writes on every -#: household. -SOURCE_HOUSEHOLD_KEY_COLUMN = "source_household_key" -#: The source columns the step writes, which a frame entering it must lack. -UK_GEOGRAPHIC_SOURCE_COLUMNS = ( - SOURCE_HOUSEHOLD_ID_COLUMN, - SOURCE_YEAR_COLUMN, - SOURCE_HOUSEHOLD_KEY_COLUMN, -) +# The source columns the step writes (UK_GEOGRAPHIC_SOURCE_COLUMNS, with +# SOURCE_YEAR_COLUMN and SOURCE_HOUSEHOLD_KEY_COLUMN) are named in the +# geographic_clones leaf, which the stages that read them hash without this +# module, and re-exported here. #: Every column the step adds: each table's clone index, then the household's #: source columns. UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS = ( @@ -128,6 +139,19 @@ ) #: How the clones share a household's weight: each takes an equal share. GEOGRAPHIC_EXPAND_WEIGHT_SPLIT = "equal" +#: The operation the stage declares, and the declaration's words for the +#: clones' order and for the household total, which the 1/K split keeps to +#: rounding (microcosm#931). +GEOGRAPHIC_EXPAND_OPERATION_KIND = "expand_geographic_clones" +GEOGRAPHIC_EXPAND_ORDER = "clone_major" +GEOGRAPHIC_EXPAND_CONSERVATION = "household_total_to_rounding" +#: How the declaration numbers the clones' ids: the copy-id table's field and +#: its radix, ``10**9``. +GEOGRAPHIC_EXPAND_ID_REMAPPING = ( + f"copy-id table field {GEOGRAPHIC_COPY_ID_FIELD}; clone k takes every entity " + "id + k x the field's radix " + f"(10**{len(str(copy_id_table.radix(GEOGRAPHIC_COPY_ID_FIELD))) - 1})" +) def mint_geographic_clone_ids(ids: np.ndarray, k: int) -> np.ndarray: @@ -163,12 +187,38 @@ def require_room_for_geographic_clones(ids: np.ndarray, what: str) -> None: source_id_columns={"household": SOURCE_HOUSEHOLD_ID_COLUMN}, ) #: The columns a frame entering the step must lack: what the step writes, and -#: every copy step's columns, since the step runs before any copy. +#: every copy step's columns, since the step runs before any other copy step. _RESERVED_COLUMNS = tuple( dict.fromkeys((*UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, *UK_COPY_LINEAGE.copy_columns)) ) +def geographic_expand_operation_parameters() -> tuple[ + tuple[str, dict[str, object]], ... +]: + """The reviewed operation the stage must declare verbatim: what + :data:`UK_GEOGRAPHIC_CLONES` and the UK wrapper do, and no clone count.""" + + return ( + ( + GEOGRAPHIC_EXPAND_OPERATION_KIND, + { + "entity": "household", + "clone_index_columns": dict(UK_GEOGRAPHIC_CLONES.clone_index_columns), + "ordinal_bound": GEOGRAPHIC_CLONE_LIMIT, + "id_remapping": GEOGRAPHIC_EXPAND_ID_REMAPPING, + "order": GEOGRAPHIC_EXPAND_ORDER, + "weight_split": GEOGRAPHIC_EXPAND_WEIGHT_SPLIT, + "weight_kind_out": WeightKind.DESIGN.value, + "source_columns": list(UK_GEOGRAPHIC_SOURCE_COLUMNS), + "conservation": GEOGRAPHIC_EXPAND_CONSERVATION, + "declared_factor": 1.0, + "reason": GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON, + }, + ), + ) + + @dataclass(frozen=True) class UKGeographicExpandResult: """The clones and what the step's evidence reads of them. @@ -210,8 +260,10 @@ def _clone_count(clones: object) -> int: def _assert_geographic_expand_stage(stage: SourceStageSpec) -> None: - """The stage a transform runs as: the step's name, household grain, and - exactly the step's columns as outputs, with no rewrites.""" + """The stage a transform runs as: the step's name, household grain, + exactly the step's columns as outputs with no rewrites, the reviewed + operation (:func:`geographic_expand_operation_parameters`) and no + artifact.""" if not isinstance(stage, SourceStageSpec): raise TypeError("The geographic expansion runs as a SourceStageSpec stage.") @@ -233,6 +285,12 @@ def _assert_geographic_expand_stage(stage: SourceStageSpec) -> None: f"{UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS} and no rewrites, got " f"{tuple(stage.outputs)} / {tuple(stage.rewrites)}." ) + _assert_closed_world_operations(stage, geographic_expand_operation_parameters()) + if tuple(stage.artifacts): + raise ValueError( + "The geographic expansion reads no artifact; the stage must declare " + f"none, got {len(stage.artifacts)}." + ) def _refuse_other_weighted_entities(frame: Frame) -> None: @@ -273,13 +331,14 @@ def _refuse_carried_lineage(frame: Frame) -> None: found = [column for column in _RESERVED_COLUMNS if column in carried] if found: raise ValueError( - "The geographic expansion clones a frame once, before any copy " - f"step; the frame already carries {found}." + "The geographic expansion clones a frame once, before any other " + f"copy step; the frame already carries {found}." ) def _source_year(frame: Frame) -> int: - """The frame's time period as a year, as the SPI support channel reads it.""" + """The frame's time period as a year: the source year the SPI support + channel wrote before this stage took the column over.""" try: return int(uk_time_period(frame)) @@ -295,10 +354,11 @@ def _with_source_year_and_key(frame: Frame, source_year: int) -> Frame: ``source_household_key``, before any clone exists. The key is written as Python strings in an object column whatever pandas - infers for text: the dtype the SPI support channel's own write gives with - string inference off, as in the legacy plan, so that plan's object-string - columns stay what they were. The graph casts the column to its declared - string dtype. + infers for text: the dtype the SPI support channel's own write gave before + this stage took the column over, as it still does on a household table + without the clone index, with string inference off, as in the legacy + plan, so that plan's object-string columns stay what they were. The graph + casts the column to its declared string dtype. """ household = frame.table("household").copy() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py index 79fb8f50b5..cb7ecdf719 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py @@ -16,7 +16,10 @@ vintage for is refused. - **Dtype.** The key is written in the graph's ``string`` dtype (pandas' ``StringDtype("python")``), as ``uk.full.identity`` writes it, so it is not - a legacy object-string column. + a legacy object-string column. The step names the dtype itself, as + ``atomic_household_lineage`` does, rather than through the graph's token + table: a stage's transform computes through the Frame's modules alone, never + the executor's. So clone 0's key is the key the dense identity gives the same base household, clone ``k``'s the key it gives that household's clone ``k``, and growing K @@ -28,7 +31,11 @@ once), that the keys are unique, the source and the vintages; no key is ever printed. -No stage runs this step yet, so it moves no output and no node key. +The spine's ``geographic_identity`` stage runs this step, right after +``geographic_expand``. Its declared operation, ``key_households_by_clone_path``, +must be the reviewed one (:func:`geographic_identity_operation_parameters`), +and its one artifact the FRS release, with ``runtime_sha256_required``, whose +bytes enter the stage's contract. """ from __future__ import annotations @@ -42,11 +49,14 @@ from microcosm.build.source_lineage import copy_paths from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.uk_runtime.atomic_area_support import IDENTITY_COLUMN from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations from microcosm.build.uk_runtime.frs_release import UKFRSRelease, load_uk_frs_release -from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column -from microcosm.build.uk_runtime.geographic_expand import SOURCE_YEAR_COLUMN +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + SOURCE_YEAR_COLUMN, + ladder_clone_index_column, +) from microcosm.build.uk_runtime.household_lineage import ( GEOGRAPHIC_SUPPORT_BRANCH, UK_COPY_LINEAGE, @@ -56,18 +66,21 @@ distinct_carriers, rounded_count, ) -from microcosm.frame import Frame +from microcosm.frame import Frame, Weights from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN -from microcosm.graph.population import dtype_for_token __all__ = [ + "FRS_RELEASE_ARTIFACT_ROLE", + "FRS_RELEASE_RESOURCE", "GEOGRAPHIC_IDENTITY_INPUTS", + "GEOGRAPHIC_IDENTITY_OPERATION_KIND", "GEOGRAPHIC_IDENTITY_SOURCE", "GEOGRAPHIC_IDENTITY_STAGE_NAME", "UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS", "UKGeographicIdentityResult", "UKGeographicIdentityStageTransform", "frs_vintages_by_source_year", + "geographic_identity_operation_parameters", "key_uk_households_by_clone_path", ] @@ -82,7 +95,30 @@ SOURCE_YEAR_COLUMN, ladder_clone_index_column("household"), ) -UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS = (IDENTITY_COLUMN,) +UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS = (GEOGRAPHY_HOUSEHOLD_KEY_COLUMN,) +#: The operation the stage declares (microcosm#931). +GEOGRAPHIC_IDENTITY_OPERATION_KIND = "key_households_by_clone_path" +#: The stage's one artifact: the packaged FRS release, which pins the vintage +#: each key names, so its bytes enter the stage's contract. +FRS_RELEASE_ARTIFACT_ROLE = "frs_release" +FRS_RELEASE_RESOURCE = "frs_release.json" + + +def geographic_identity_operation_parameters() -> tuple[ + tuple[str, dict[str, object]], ... +]: + """The reviewed operation the stage must declare verbatim: the source the + keys name and the resource their vintage comes from.""" + + return ( + ( + GEOGRAPHIC_IDENTITY_OPERATION_KIND, + { + "source": GEOGRAPHIC_IDENTITY_SOURCE, + "vintage_resource": FRS_RELEASE_RESOURCE, + }, + ), + ) def frs_vintages_by_source_year( @@ -93,9 +129,11 @@ def frs_vintages_by_source_year( The release pins one vintage, which the loader checks starts with its survey year. The geographic expansion writes each household's ``source_year`` from the frame's time period, the release's base year, as - the SPI support channel does. So both years map to the vintage (one year - when they are equal), as the dense identity keys every household with - it; any other year maps to none. ``release`` defaults to the packaged pin. + the SPI support channel did before that stage took the column over (and + still does on a household table without the clone index). So both years + map to the vintage (one year when they are equal), as the dense identity + keys every household with it; any other year maps to none. ``release`` + defaults to the packaged pin. """ release = load_uk_frs_release() if release is None else release @@ -130,8 +168,10 @@ def evidence(self) -> dict[str, object]: def _assert_geographic_identity_stage(stage: SourceStageSpec) -> None: - """The stage a transform runs as: the step's name, household grain, and - exactly the key as its output, with no rewrites.""" + """The stage a transform runs as: the step's name, household grain, + exactly the key as its output with no rewrites, the reviewed operation + (:func:`geographic_identity_operation_parameters`), and the FRS release as + its one artifact, with ``runtime_sha256_required``.""" if not isinstance(stage, SourceStageSpec): raise TypeError("The geography identity runs as a SourceStageSpec stage.") @@ -152,6 +192,22 @@ def _assert_geographic_identity_stage(stage: SourceStageSpec) -> None: f"declare outputs {UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS} and no " f"rewrites, got {tuple(stage.outputs)} / {tuple(stage.rewrites)}." ) + _assert_closed_world_operations(stage, geographic_identity_operation_parameters()) + releases = [ + artifact + for artifact in stage.artifacts + if artifact.get("role") == FRS_RELEASE_ARTIFACT_ROLE + and artifact.get("resource") == FRS_RELEASE_RESOURCE + and artifact.get("runtime_sha256_required") is True + ] + if len(stage.artifacts) != 1 or len(releases) != 1: + raise ValueError( + "The geography identity reads its vintages from one artifact, role " + f"{FRS_RELEASE_ARTIFACT_ROLE!r} and resource {FRS_RELEASE_RESOURCE!r} " + "with runtime_sha256_required: true; the stage must declare it and no " + f"other, got {len(stage.artifacts)} artifact(s), {len(releases)} of " + "them that one." + ) def _require_int64(household: pd.DataFrame, column: str) -> np.ndarray: @@ -204,10 +260,10 @@ def key_uk_households_by_clone_path( raise TypeError("The geography identity keys a Frame.") validate_uk_national_frame(frame) household = frame.table("household") - if IDENTITY_COLUMN in household.columns: + if GEOGRAPHY_HOUSEHOLD_KEY_COLUMN in household.columns: raise ValueError( - f"The frame already carries {IDENTITY_COLUMN!r}; the geography identity " - "keys each household once." + f"The frame already carries {GEOGRAPHY_HOUSEHOLD_KEY_COLUMN!r}; the " + "geography identity keys each household once." ) _require_int64(household, SOURCE_HOUSEHOLD_ID_COLUMN) years = _require_int64(household, SOURCE_YEAR_COLUMN) @@ -231,15 +287,19 @@ def key_uk_households_by_clone_path( "every household needs its own source and clone path (microcosm#931)." ) keyed = household.copy() - keyed[IDENTITY_COLUMN] = pd.Series( - keys, index=household.index, dtype=dtype_for_token("string") + keyed[GEOGRAPHY_HOUSEHOLD_KEY_COLUMN] = pd.Series( + keys, index=household.index, dtype=pd.StringDtype(storage="python") ) tables = {entity: frame.table(entity) for entity in frame.entities} tables["household"] = keyed + # The household weights pass through unchanged. Annotated so the stage's + # static walk meets microcosm.frame's Weights module, as + # test_the_core_modules_are_on_every_nodes_path requires. + weights: Weights = frame.weights_for("household") result = Frame( tables, frame.schema, - {"household": frame.weights_for("household")}, + {"household": weights}, frame.strata, mass_log=frame.mass_log, metadata=frame.metadata, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py index 65d01c1d6e..eb72cee01a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py @@ -16,6 +16,7 @@ import hashlib import json +import operator from collections.abc import Iterable, Mapping from dataclasses import dataclass from importlib import metadata @@ -48,6 +49,7 @@ UK_SPINE_STRUCTURAL_STAGES = frozenset( { + "geographic_expand", "spi_support_channel", "spi_income_band_donors", "high_value_home_support", @@ -59,7 +61,9 @@ ) #: The last stage of the assembled checkpoint: everything through the base -#: FRS mapping and the stochastic draws. A name, not an index — a position +#: FRS mapping and the stochastic draws, with the HMRC leaves, the annual +#: income spells and the geographic clones and their keys, which run before +#: the draws (microcosm#931). A name, not an index — a position #: standing in for a key is correct only while two independently-maintained #: orderings happen to agree (the uk-data#468 class). The graph freezes the #: checkpoint after it, and the graph's spine gates and the plan path read it, @@ -83,6 +87,9 @@ # person-mass ledger the executor verifies and asserts the household-mass # invariant itself (``UKExpandStageKernel``). _STRUCTURAL_MASS = { + # The geographic clones copy every household K ways at an equal share of + # its weight (microcosm#931), so person mass is conserved, to rounding. + "geographic_expand": "conserve", "spi_support_channel": "declared", # The reserved income rows are funded from the incumbent households of # each donor's region (microcosm#1063): household mass is conserved @@ -101,6 +108,9 @@ } _STRUCTURAL_WEIGHT_KIND = { + # The clones keep the root's design weights: an EXPAND may carry its + # base's kind (microcosm#931). + "geographic_expand": "design", "spi_support_channel": "importance", "spi_income_band_donors": "importance", "high_value_home_support": "importance", @@ -234,6 +244,15 @@ "hmrc_income": "Pinned local HMRC income facts workbook.", } +#: The geographic clones' lineage: each household's true source and its clone. +#: frs_take_up, frs_person_draws and frs_household_draws run on every clone, +#: after geographic_expand, and count their receipts' support before the +#: copies, which the shared reader does on the source column wherever the +#: frame carries it, refusing the clone index without it (microcosm#931). +_GEOGRAPHIC_CLONE_LINEAGE = frozenset( + {("household", "source_household_id"), ("household", "household_clone_index")} +) + # ``None`` means the implementation genuinely has an open formula/model # surface and therefore receives every live cell. Finite sets are the direct # Frame reads inventoried from the existing transform. Structural ids, @@ -300,7 +319,8 @@ "months_since_last_birthday", ) ) - | frozenset({("benunit", "is_married")}), + | frozenset({("benunit", "is_married")}) + | _GEOGRAPHIC_CLONE_LINEAGE, # is_apprentice draws employees aged 21 to 24 at a rate set by the # frame's weights (microcosm#1372), outside full-time education (Vahid's # review of microcosm#1535, nit 3), and its receipt reads their pay for @@ -313,8 +333,9 @@ "is_in_full_time_education", "employment_income", ) - ), - "frs_household_draws": frozenset(), + ) + | _GEOGRAPHIC_CLONE_LINEAGE, + "frs_household_draws": _GEOGRAPHIC_CLONE_LINEAGE, "frs_brma": None, "was_wealth": None, # The LISA model reads the was_wealth household predictors through the @@ -339,7 +360,8 @@ # channel and the household reference person's sub-letting rent), the # landlords' ages the FRS-concept finance-cost draw conditions on, and the # rental property values the coherence receipt reads (microcosm#1106, - # microcosm#840). + # microcosm#840). The geographic clone index is declared too, so that a + # check of the stage can read its frame's clone count (microcosm#931). "property_components": frozenset( { ("person", "age"), @@ -351,6 +373,7 @@ ("household", "subrent"), ("household", "other_residential_property_value"), ("household", "non_residential_property_value"), + ("household", "household_clone_index"), } ), # Engine-free: the owners' region, tenure, value and wealth, the support @@ -397,6 +420,18 @@ "gender", ) ), + # The geographic clones copy whole households, so their surface is open + # (microcosm#931). + "geographic_expand": None, + # The geography key reads each household's true source, its source year + # and its clone, the lineage geographic_expand writes (microcosm#931). + "geographic_identity": frozenset( + { + ("household", "source_household_id"), + ("household", "source_year"), + ("household", "household_clone_index"), + } + ), "spi_support_channel": None, "spi_income_band_donors": None, "hmrc_spi_income_spine": None, @@ -421,6 +456,7 @@ ("household", "household_support_channel"), ("household", "household_support_clone_index"), ("household", "source_household_id"), + ("household", "household_clone_index"), } ), # Engine-free: the sub-major predictors, the major group it nests in, the @@ -449,6 +485,7 @@ { ("household", "source_household_id"), ("household", "household_support_clone_index"), + ("household", "household_clone_index"), ("person", "age"), ("person", "is_household_head"), ("person", "employment_income"), @@ -547,6 +584,7 @@ ("household", "source_household_id"), ("household", "household_support_channel"), ("household", "household_support_clone_index"), + ("household", "household_clone_index"), ("household", "household_is_high_value_support_copy"), ("household", "high_value_support_copy_index"), ("person", "is_benunit_head"), @@ -844,7 +882,12 @@ def owned(self, *, rewrite: bool = False) -> Owned: "accommodation_type", "council_tax_band", } -_ROOT_HOUSEHOLD_INT = {"num_bedrooms", "council_tax_single_adult_raw"} +#: imd_decile and council_tax_band_observed are the root's two area- +#: conditioning carriers (frs_spine.FRS_AREA_CONDITIONING_COLUMNS, +#: microcosm#931): the decile int64, 0 where none is observed, and the flag +#: bool. +_ROOT_HOUSEHOLD_INT = {"num_bedrooms", "council_tax_single_adult_raw", "imd_decile"} +_ROOT_HOUSEHOLD_BOOL = {"council_tax_band_observed"} _STRUCTURAL_COLUMNS = { "person_id", "person_benunit_id", @@ -865,6 +908,8 @@ def _root_cells(stage_outputs: Iterable[str]) -> tuple[_Cell, ...]: cells.append(_Cell("household", column, "string")) elif column in _ROOT_HOUSEHOLD_INT: cells.append(_Cell("household", column, "int64")) + elif column in _ROOT_HOUSEHOLD_BOOL: + cells.append(_Cell("household", column, "bool")) elif column in _ROOT_PERSON_STRING: cells.append(_Cell("person", column, "string")) elif column in _ROOT_PERSON_BOOL: @@ -1215,6 +1260,22 @@ def _cells( "contributory_benefits_annual_share", ), ), + # Each table's geographic clone index and each household's source + # columns (microcosm#931), in the order the transform writes them: the + # source year and key on the base households, then the shared core's + # source id and clone index. The graph appends an EXPAND's new cells in + # this order, so it matches the legacy plan's column order. + "geographic_expand": ( + _Cell("person", "person_clone_index", "int64"), + _Cell("benunit", "benunit_clone_index", "int64"), + _Cell("household", "source_year", "int64"), + _Cell("household", "source_household_key", "string"), + _Cell("household", "source_household_id", "int64"), + _Cell("household", "household_clone_index", "int64"), + ), + "geographic_identity": (_Cell("household", "geography_household_key", "string"),), + # The households' source columns are geographic_expand's cells; the + # channel's copies inherit them (microcosm#931). "spi_support_channel": ( _Cell("person", "person_source_id", "int64"), _Cell("person", "person_support_channel", "string"), @@ -1222,9 +1283,6 @@ def _cells( _Cell("benunit", "benunit_source_id", "int64"), _Cell("benunit", "benunit_support_channel", "string"), _Cell("benunit", "benunit_support_clone_index", "int64"), - _Cell("household", "source_household_id", "int64"), - _Cell("household", "source_year", "int64"), - _Cell("household", "source_household_key", "string"), _Cell("household", "household_source_id", "int64"), _Cell("household", "household_support_channel", "string"), _Cell("household", "household_support_clone_index", "int64"), @@ -1708,6 +1766,11 @@ def _numerical_dependency_versions() -> tuple[tuple[str, str], ...]: The missing marker permits source-only declarations without engine extras; installing the engine then produces a different identity, never a false hit. + microcosm-build does not declare microdf-python, which policyengine-uk + installs, but private_school_attendance stores its own restatement of the + engine's percentile rank, by microdf's weighted percentile rank, wherever + the engine computes the frame clone by clone, so its version is bound too + (microcosm#931; it is declared at the next lock bump). """ versions = [] for name in ( @@ -1717,6 +1780,7 @@ def _numerical_dependency_versions() -> tuple[tuple[str, str], ...]: "pandas", "scikit-learn", "quantile-forest", + "microdf-python", ): try: version = metadata.version(name) @@ -1732,10 +1796,21 @@ def uk_spine_graph( source_mode: str = "bundle", sample_fraction: float = 1.0, sample_seed: int = UK_SAMPLE_SEED_DEFAULT, + geographic_clones: int = 1, ) -> Graph: - """Return the source-bound graph for the packaged UK FRS spine.""" + """Return the source-bound graph for the packaged UK FRS spine. + + ``geographic_clones`` is the spine's clone count ``K``, 1 to the + ``geographic_support`` copy-id field's bound. The geographic_expand node + carries it as a parameter, so it enters that node's key, and through the + chain every later node's, but never a stage contract (microcosm#931). + """ from .frs_spine import UKFRSSpineStageTransform + from .geographic_expand import ( + GEOGRAPHIC_CLONE_LIMIT, + GEOGRAPHIC_EXPAND_STAGE_NAME, + ) if source_mode not in {"bundle", "split"}: raise ValueError("UK graph source_mode must be 'bundle' or 'split'.") @@ -1743,8 +1818,29 @@ def uk_spine_graph( raise ValueError("UK graph sample_fraction must be in (0, 1].") if sample_seed < 0: raise ValueError("UK graph sample_seed must be non-negative.") + try: + clones = ( + None + if isinstance(geographic_clones, bool) + else operator.index(geographic_clones) + ) + except TypeError: + clones = None + if clones is None or not 1 <= clones <= GEOGRAPHIC_CLONE_LIMIT: + raise ValueError( + "UK graph geographic_clones must be a whole number from 1 to " + f"{GEOGRAPHIC_CLONE_LIMIT}, the geographic_support copy-id field's " + "bound." + ) resolved = load_country_spec("uk") if spec is None else spec stages = _manifest_stages(resolved) + if clones != 1 and not any( + stage.stage == GEOGRAPHIC_EXPAND_STAGE_NAME for stage in stages + ): + raise ValueError( + "UK graph geographic_clones above 1 needs the " + f"{GEOGRAPHIC_EXPAND_STAGE_NAME!r} stage, which makes the clones." + ) dependency_versions = _numerical_dependency_versions() # The root transform loads the complete national-frame seed schema even # when a reduced hermetic manifest names only the output under test. @@ -1828,6 +1924,13 @@ def uk_spine_graph( "stage_contract_sha256": _stage_contract_sha256( manifest_stage, resolved ), + # The clone count K is the node's, never its stage + # contract's (microcosm#931). + **( + {"geographic_clones": clones} + if stage_name == GEOGRAPHIC_EXPAND_STAGE_NAME + else {} + ), }, structural=StructuralDelta.EXPAND, artifact_outputs=( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 7223741120..faa19d4f5e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -60,6 +60,7 @@ from .. import ( chronicle_epoch, + clone_expansion, gates, ledger_targets, raking, @@ -72,6 +73,7 @@ from . import ( advani_summers, annuity_trust_column, + atomic_household_identity, birth_month, bus_fare_pricing, bus_support_pricing, @@ -134,6 +136,7 @@ uc_relationships, was_wealth, ) +from .graph import UK_SPINE_STRUCTURAL_STAGES from .graph_frames import cast_owned, normalize_create_frame from .national_frame import UK_NATIONAL_SCHEMA @@ -175,6 +178,8 @@ "etb_services": "etb_services", "frs_hmrc_spine_leaves": "spi_spine", "frs_annual_income_spells": "frs_annual_income", + "geographic_expand": "geographic_expand", + "geographic_identity": "geographic_identity", "spi_support_channel": "spi_spine", "spi_income_band_donors": "spi_band_donors", "hmrc_spi_income_spine": "spi_spine", @@ -327,6 +332,13 @@ *_PINNED_TAB_MODULES, stochastic_assignment, ), + # The geographic clones are the shared clone expansion core's, under the + # UK's declaration, whose ids are minted in the copy-id table's + # geographic_support field (microcosm#931). + "geographic_expand": (clone_expansion, copy_id_table), + # The geography key is household_draw_key, encoded canonically, over the + # FRS vintage the release pins for each source year (microcosm#931). + "geographic_identity": (atomic_household_identity, frs_release, graph_canonical), # The SPI income stage's imputation, band-donor resample and receipt live # in spi_income, so an edit there moves the stage's hash (microcosm#1278), # and its twin transport in spi_rank_transport (microcosm#840). The @@ -377,13 +389,16 @@ # trace's digest, ages people by birth month, caps pension contributions # with their receipt, and codes council tax bands in the shared band order # (microcosm#931). It writes the annuity and trust descriptor under the - # name annuity_trust_column holds (microcosm#931, ruled 9 Nov (late)). + # name annuity_trust_column holds (microcosm#931, ruled 9 Nov (late)). It + # reads each household's nation, for its IMD decile and its band flag, + # from the region tier through council_tax_constants (microcosm#931). "frs_spine": ( uc_relationships, annuity_trust_column, copy_id_table, birth_month, council_tax_constants, + geography_constants, pension_coherence, receipt_counts, trace, @@ -868,6 +883,11 @@ # (microcosm#931). "frs_hmrc_spine_leaves", "frs_annual_income_spells", + # The geographic clones and their keys count their evidence in true + # source households, and the keys read each clone path through the + # shared lineage reader (microcosm#931). + "geographic_expand", + "geographic_identity", "hmrc_spi_income_spine", "spi_support_channel", "spi_income_band_donors", @@ -1208,7 +1228,7 @@ def _fixture_descriptor( missing = sorted(set(_STAGE_MODULES) - set(stages)) extra = sorted(set(stages) - set(_STAGE_MODULES)) raise ValueError( - "UK parity fixture must describe the current 39-stage spine " + "UK parity fixture must describe every stage of the current spine " f"(missing={missing}, extra={extra})." ) return descriptor, stages @@ -1247,6 +1267,8 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: from .frs_person_draws import UKFRSPersonDrawsStageTransform from .frs_relationships import UKFRSRelationshipsStageTransform from .frs_take_up import UKFRSTakeUpStageTransform + from .geographic_expand import UKGeographicExpandStageTransform + from .geographic_identity import UKGeographicIdentityStageTransform from .high_value_home_support import UKHighValueHomeSupportStageTransform from .isa_subscriptions import UKISASubscriptionsStageTransform from .lcfs_consumption import UKLCFSConsumptionStageTransform @@ -1354,7 +1376,14 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: ) ) - engine = PolicyEngineUKEngine() + # The fixture's spine clones every household geographic_clones times, and + # its engine refuses a frame of any other clone count (microcosm#931). + geographic_clones = config.get("geographic_clones") + if isinstance(geographic_clones, bool) or not isinstance(geographic_clones, int): + raise ValueError( + "UK parity fixture config.geographic_clones must be an integer clone count." + ) + engine = PolicyEngineUKEngine(clones=geographic_clones) contract = load_uk_take_up_contract() qrf_estimators = int(config["qrf_estimators"]) donor_sample_size = int(config["spi_donor_sample_size"]) @@ -1451,6 +1480,12 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: "frs_annual_income_spells": UKFRSAnnualIncomeStageTransform( raw_dir, stage=stages["frs_annual_income_spells"] ), + "geographic_expand": UKGeographicExpandStageTransform( + stage=stages["geographic_expand"], clones=geographic_clones + ), + "geographic_identity": UKGeographicIdentityStageTransform( + stage=stages["geographic_identity"] + ), "spi_support_channel": UKSPISupportChannelStageTransform( stage=stages["spi_support_channel"], sample_fraction=sample_fraction, @@ -1881,6 +1916,7 @@ def _expand_cells(context: KernelContext) -> tuple[tuple[str, str, str], ...]: #: ``{entity}_source_id`` instead. _COPY_ID_FIELDS = MappingProxyType( { + "geographic_expand": "geographic_support", "high_value_home_support": "high_value_support", "cgt_support_split": "cgt_support_split", "cgt_incidence_clone": "cgt_incidence_clone", @@ -2029,6 +2065,16 @@ def run(self, context: KernelContext) -> KernelResult: f"UK stage {self.stage!r} has no bound production transform; " "recorded fixture deltas are not accepted as parity evidence." ) + # The geographic clone count is a parameter of the node that makes the + # clones, so it is in that node's key, and the transform must make + # exactly that many (microcosm#931); no other stage has either. + declared_clones = context.params.get("geographic_clones") + transform_clones = getattr(transform, "clones", None) + if transform_clones != declared_clones: + raise ValueError( + f"UK stage {self.stage!r} declares {declared_clones!r} geographic " + f"clone(s) on its node, but its transform makes {transform_clones!r}." + ) before = _minimal_frame(context) after = _invoke_transform(transform, before, context) if not isinstance(after, Frame): @@ -2166,15 +2212,7 @@ def build_uk_registry( fixture_resolver = _FixtureTransformResolver() for stage in sorted(stage_names): transform = implementations.get(stage) - if stage in { - "spi_support_channel", - "spi_income_band_donors", - "high_value_home_support", - "cgt_support_split", - "cgt_incidence_clone", - "cgt_incidence_anchor", - "cgt_residential_split", - }: + if stage in UK_SPINE_STRUCTURAL_STAGES: registry.register( UKExpandStageKernel( stage, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index bdeb66ed25..5f05ed38ac 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -1102,6 +1102,7 @@ def implementation_hash(self) -> str: donor_uprating, etb_services, gate_scopes, + geographic_clones, graph_population, graph_targets, national_frame, @@ -1119,7 +1120,10 @@ def implementation_hash(self) -> str: # manifest's effective sample size (calibrate.solve), the admin # totals' derived anchors (etb_services), the CGT projection # (cgt_projection, donor_uprating) and the canonical encoding of its - # artifacts (microcosm#931). + # artifacts (microcosm#931). Its support-family fold reads the + # geographic clone index columns the national frame carries from a + # clone-first spine, by the names geographic_clones gives them + # (diagnostics.uk_support_family_geographic_clone_column). return hashlib.sha256( canonical_json( { @@ -1134,6 +1138,7 @@ def implementation_hash(self) -> str: # The input-mass gate's source-household counts. receipt_counts, diagnostics, + geographic_clones, support_floor_rules, gates, calibrate_artifacts, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index ffdd61e97f..b432fab704 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -92,8 +92,11 @@ GEOGRAPHY_ASSIGNMENTS = ("atomic", "legacy") #: Lineage columns the post-clone household identity is keyed on. The support #: channel, the CGT support split's copy flag and index (microcosm#1045) and -#: the clone flag come from the spine stages; the geographic clone index from -#: ``uk.full.expand``. Calibration K, weights and remapped ids are not inputs. +#: the clone flag come from the spine stages, as do the source id and the +#: geographic clone index on a clone-first spine, whose ``geographic_expand`` +#: stage writes them (microcosm#931); on a spine built before that stage, +#: ``uk.full.expand`` writes the index. Calibration K, weights and remapped ids +#: are not inputs. UK_GEOGRAPHY_IDENTITY_INPUTS = UK_HOUSEHOLD_LINEAGE_COLUMNS PROPERTY_LOCALISATION_MODES = property_localisation.PROPERTY_LOCALISATION_MODES PROPERTY_QUANTILE_TYPE = ArtifactType("microcosm.uk.property-quantile-draw", 1) @@ -1383,12 +1386,22 @@ def _atomic_geography_nodes( if ("household", column) in cells ) ) + # A clone-first spine already keys its households (geographic_identity), + # so there the node rewrites the key on the pool (microcosm#931). + identity_cell = ("household", atomic_area_support.IDENTITY_COLUMN) identity = Node( id="uk.full.identity", kernel=UKGeographyIdentityKernel.ref, population="uk.full.expand", inputs=(Slice("household", lineage_inputs),), - outputs=(Owned("household", atomic_area_support.IDENTITY_COLUMN, "string"),), + outputs=( + Owned( + "household", + atomic_area_support.IDENTITY_COLUMN, + "string", + rewrite=identity_cell in cells, + ), + ), params={ **common, "source": identity_source, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py index d8d62f33a5..ff6297a668 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py @@ -43,8 +43,10 @@ #: Lineage columns the post-clone household identity is keyed on. The support #: channel, the CGT support split's copy flag and index (microcosm#1045), the #: high-value home support copy flag and index (microcosm#1240) and the clone -#: flags come from the spine stages; the geographic clone index from -#: ``uk.full.expand``. Calibration K, weights and remapped ids are not inputs. +#: flags come from the spine stages. So do the source id and the geographic +#: clone index, which the spine's ``geographic_expand`` stage writes +#: (microcosm#931); on a spine built before that stage, ``uk.full.expand`` +#: writes the index. Calibration K, weights and remapped ids are not inputs. HIGH_VALUE_SUPPORT_COPY_COLUMN = "household_is_high_value_support_copy" HIGH_VALUE_SUPPORT_INDEX_COLUMN = "high_value_support_copy_index" UK_HOUSEHOLD_LINEAGE_COLUMNS: tuple[str, ...] = ( @@ -106,8 +108,8 @@ #: The branches every post-clone household carries: the SPI support channel #: and the CGT steps. High-value support (a spine built before that stage has -#: neither of its columns) and the geographic clone index (a spine frame has -#: none yet) are read where present. +#: neither of its columns) and the geographic clone index (a spine built +#: before ``geographic_expand`` has none) are read where present. _REQUIRED_BRANCHES = ( "spi_support_channel", "cgt_support_split", @@ -171,8 +173,8 @@ def fold_geographic_clones( other copy (SPI support, high-value support and the CGT steps) stays a row of its own (microcosm#931). The record's key is its true source and its clone path without the geographic branch, through the shared reader - (``microcosm.build.source_lineage``); a frame without geographic clones, - as every spine is today, keeps ``rows`` and ``weights`` value for value. + (``microcosm.build.source_lineage``); a frame of one clone, with or + without the clone index, keeps ``rows`` and ``weights`` value for value. The fold orders a key's clones by ``entity``'s id column. Their ids differ only by each clone's offset (the copy-id table's diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py index 1c39b2fcbc..6a45751f89 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py @@ -790,6 +790,14 @@ def _write_uk_single_year_tables( # and the split reads them); the engine sums the person balances. "cash_isa", "stocks_and_shares_isa", + # The root's two area-conditioning carriers + # (frs_spine.FRS_AREA_CONDITIONING_COLUMNS, microcosm#931): each + # household's IMD decile within its nation and whether its council + # tax band is a band the FRS gives it that its region uses. The + # clone-first spine's area assignment reads them; they are no engine + # inputs. + "imd_decile", + "council_tax_band_observed", ), } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py index 026fbe563f..5a13e1d0ca 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py @@ -23,14 +23,14 @@ in the pinned engine. The rank is over the people a simulation holds, so a simulation over part of -the frame ranks that part alone. Once a dense spine's households carry more -than one geographic clone, its engine calls run clone by clone +the frame ranks that part alone. Once a spine's households carry more than +one geographic clone, its engine calls run clone by clone (microcosm#931), so the stage restates the formula's last step itself, over every person in the frame (:func:`restate_private_school_attendance`): from engine values that each household's own records decide, the engine's own parameters, and microdf's weighted percentile rank, the rank the formula calls. Where the engine computes the frame in one simulation (the national -line, and a spine before its clones exist) the stage stores the engine's +line, and a spine of one geographic clone) the stage stores the engine's flags once the restatement equals them bit for bit, so every such build re-checks the restatement on its own data; where it computes the frame clone by clone the stage stores the restatement. The receipt names the route and diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/receipt_counts.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/receipt_counts.py index 32cb1225fa..9d1947989a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/receipt_counts.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/receipt_counts.py @@ -84,10 +84,13 @@ def record_source_households( (``microcosm.build.source_lineage.record_sources``) on the UK's copy lineage. A frame without the column is refused, and so is a null source: it drops out of a distinct count, so a cell would read fewer sources than - it rests on. A stage that runs before the SPI support channel, the first - copying step, says so with ``before_copies``: no row is a copy yet, so - each household is its own source and its id serves, unless the frame - already carries copy lineage. + it rests on. A stage that can run before the first copying step says so + with ``before_copies``: on a frame without the column and without copy + lineage, no row is a copy yet, so each household is its own source and its + id serves. On the clone-first spine the first copying step is + ``geographic_expand``, which writes the column, so a stage after it that + says so, ``frs_take_up`` for one, reads it and counts a household's + geographic clones once (microcosm#931). """ if not isinstance(tables, Mapping): @@ -110,8 +113,9 @@ def own_source_households(person: pd.DataFrame) -> np.ndarray: """Each person's own household as its true source: the rule before copies. For a caller that holds a person table without the frame's lineage, as - every household is its own source before the SPI support channel copies - any; a frame with copies counts through :func:`record_source_households`. + every household is its own source before the first copying step, + ``geographic_expand`` on the clone-first spine (microcosm#931); a frame + with clones or copies counts through :func:`record_source_households`. """ return person[UK_NATIONAL_SCHEMA.membership_column("household")].to_numpy() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index b7ea59e5ad..dae519744d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -22,9 +22,11 @@ # lineage read the clone index columns' names through household_lineage and # hash that small module, not this one, and the full build's population and # export kernels hash it beside this one. Kept importable from here for this -# module's other callers. +# module's other callers. The households' source columns, which the +# clone-first spine's geographic_expand writes, are named there too. from microcosm.build.uk_runtime.geographic_clones import ( ARTIFACT_CLONE_INDEX_COLUMN, + UK_GEOGRAPHIC_SOURCE_COLUMNS, ladder_clone_index_column, ) from microcosm.build.uk_runtime.geography_ladder import ( @@ -112,9 +114,12 @@ def refuse_dense_clone_id_overflow( a production national spine does, leaves no int64 room for a second clone. This refuses wherever the dense offset would overflow int64; one clone, and any spine whose ids leave room (one without copy rows, say), - expand as before. Dense builds with more than one clone of such a spine - wait for the clone-first spine, which mints the geographic clones at its - start (#931 group (c)). The message names no id. + expand as before, as long as the spine carries no geographic clone index. + Dense builds with more than one clone of such a spine wait for the + clone-first spine, which mints the geographic clones at its start and + which the dense line takes from #931 group (c); until then the dense + expansion refuses a spine carrying geographic_expand's clone index, at + any clone count, before it reaches this check. The message names no id. """ if type(n_clones) is not int or n_clones <= 1: @@ -132,7 +137,8 @@ def refuse_dense_clone_id_overflow( "multiples of 10**18, and every dense clone is offset above the " "spine's largest id. Dense builds with more than one clone of such a " "spine wait for the clone-first spine of microcosm#931, which mints " - "the geographic clones at its start; n_clones=1 still expands." + "the geographic clones at its start; n_clones=1 still expands a " + "spine that carries no geographic clone index." ) @@ -413,8 +419,10 @@ def expand_uk_geographic_pool( # lineage) is replaced, exactly as the pre-Frame clone overwrote the # column in place — dropped here so the per-entity in-memory names # cannot collide with it at the writer's rename. The reserved - # {entity}_clone_index names have no legitimate producer and fail - # closed rather than being silently overwritten. + # {entity}_clone_index names fail closed rather than being silently + # overwritten: their one other producer is the clone-first spine's + # geographic_expand (microcosm#931), whose spine carries its geographic + # clones already and which the dense line takes only from #931 group (c). tables = { "person": person_frame, "benunit": benunit_frame, @@ -425,7 +433,11 @@ def expand_uk_geographic_pool( if reserved in table.columns: raise ValueError( f"input {entity} table already carries the reserved in-memory " - f"clone column {reserved!r}; the ladder clone owns that name." + f"clone column {reserved!r}; the ladder clone owns that name. A " + "clone-first spine, whose geographic_expand stage writes it, " + "carries its geographic clones already, and the dense line " + "builds on such a spine only from microcosm#931 group (c), at " + "any n_clones." ) if ARTIFACT_CLONE_INDEX_COLUMN in table.columns: table.drop(columns=[ARTIFACT_CLONE_INDEX_COLUMN], inplace=True) @@ -772,9 +784,32 @@ def _attach_source_lineage( *, source_year: int | None, ) -> pd.DataFrame: - """Default the immediate-layer lineage columns the crosswalk route adds.""" + """Default the immediate-layer lineage columns the crosswalk route adds. + + Only on a household table without the geographic clone index: on the + clone-first spine, geographic_expand writes every household's source + columns and a clone's own ids are not its source, so a table carrying + the index must carry all three and is refused, by column name alone, if + it lacks any (microcosm#931). The reserved-column refusal before it + already refuses every such table, so this refusal is a second guard. + """ frame = household + clone_index = ladder_clone_index_column("household") + if clone_index in frame.columns: + missing = [ + column + for column in UK_GEOGRAPHIC_SOURCE_COLUMNS + if column not in frame.columns + ] + if missing: + raise ValueError( + f"The household table carries {clone_index!r} but not {missing}: " + "on the clone-first spine geographic_expand writes every " + "household's source columns, and the dense line never takes a " + "clone's own ids for its source (microcosm#931)." + ) + return frame if "source_household_id" not in frame.columns: frame["source_household_id"] = frame["household_id"] if "source_year" not in frame.columns and source_year is not None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py index d29f17f367..9ea0bfd893 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py @@ -169,14 +169,17 @@ #: An SPI-channel adult's pay is the tape's, and the FRS bonus it carries is #: its twin's, so the two do not belong together (Vahid's review of #1383, #: item 6). It draws a bonus share of pay from a base-FRS adult paid alike -#: instead (María's ruling of 2026-10-24); FRS rows keep their own bonus. +#: instead (María's ruling of 2026-10-24); FRS rows keep their own bonus. It +#: draws within its own geographic clone, from cells pooled over every clone +#: (microcosm#931; :func:`draw_spi_bonuses`). BONUS_SACRIFICE_SPI_DRAW = ( "an SPI-channel adult with pay draws the bonus share of pay " "(frs_bonus_amount over employment_income) of a base-FRS adult with pay in " - "the same pay cell, by person weight, keyed on person_id; the cells are the " - "base-FRS paid adults' weighted pay quantiles at spi_cell_quantiles, and a " - "cell with fewer than spi_min_donor_sources donor source households joins " - "the one below" + "the same pay cell and the same geographic clone (household_clone_index; a " + "spine without it is one clone), by person weight, keyed on person_id; the " + "cells are the base-FRS paid adults' weighted pay quantiles over every clone " + "at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources " + "donor source households joins the one below, and the lowest the one above" ) BONUS_SACRIFICE_SPI_CELL_QUANTILES = ( 0.1, @@ -355,8 +358,9 @@ def draw_spi_bonuses( (``BONUS_SACRIFICE_SPI_DRAW``); an SPI adult without pay has no bonus. Every other row keeps its own. - ``clone_indices`` gives each row's geographic clone (microcosm#931); None, - as on every spine today, puts every row in clone 0. The pay cells and the + ``clone_indices`` gives each row's geographic clone (microcosm#931), 0 + throughout a one-clone spine; None, for a frame without + ``household_clone_index``, puts every row in clone 0. The pay cells and the donor-source floor pool every clone's donors, the floor counted in distinct ``source_ids``. Each SPI adult draws from the donors of its own clone in its cell, in id order, on the same keyed uniform. So a frame of @@ -453,7 +457,8 @@ def _geographic_clone_indices( """Each person's geographic clone: its household's ``household_clone_index``. ``households`` is indexed by ``household_id``. None without the column, - which is one clone, as every spine is today (microcosm#931). The stage's + which is one clone; the spine's ``geographic_expand`` stage writes it on + every household, 0 throughout a one-clone spine (microcosm#931). The stage's geographic fold reads the column first, as the ``geographic_support`` branch's ordinals, and refuses one that is not a nonnegative integer. """ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py index 2fe980f063..d7d068e3dd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py @@ -34,8 +34,9 @@ one-clone spine's (its weights over the clone count) only while that scale is the one a one-clone spine takes: one at both, or bound by a region of clone 0. Where another clone's region binds, every clone's donors, clone 0's included, -take the lower scale. A spine without the column is one clone, as every spine -is today: ``uk.full.expand`` writes the index after the spine. +take the lower scale. The spine's ``geographic_expand`` stage writes the column +before this stage runs, 0 on every household of a one-clone spine; a household +table without it is one clone. """ from __future__ import annotations @@ -155,15 +156,15 @@ "maximum_donor_weight))" ) SPI_INCOME_BAND_DONOR_WEIGHT_RULE = ( - "published band taxpayers / (geographic clone count x band donor count), " - "equal within a band" + "published band taxpayers / (geographic clone count x one clone's band donor " + "count), equal within a band" ) SPI_INCOME_BAND_DONOR_DRAW = ( "weighted_without_replacement_identity_keyed within each geographic clone " - "(one before uk.full.expand): Efraimidis-Spirakis keys log(u) / " - "(household weight x propensity) on stable_identity_uniforms(person_id, " - "seed, salt:band); the largest keys are seated, one carrier per household, " - "the highest band first" + "(a spine without household_clone_index is one clone): Efraimidis-Spirakis " + "keys log(u) / (household weight x propensity) on " + "stable_identity_uniforms(person_id, seed, salt:band); the largest keys are " + "seated, one carrier per household, the highest band first" ) SPI_INCOME_BAND_DONOR_FUNDING = ( "stratum reallocation per geographic clone: every incumbent household of " @@ -997,10 +998,10 @@ def _funding_strata(household: pd.DataFrame) -> pd.Series: def _geographic_clone_indices(household: pd.DataFrame) -> np.ndarray | None: """Each household's geographic clone index, or None without the column. - The clone-first spine (microcosm#931) gives every household the index of - its geographic clone in ``household_clone_index``, and its copies keep - their root's; a household table without the column, as on every spine - before ``uk.full.expand``, is one clone. Refuses a non-integer column, + The clone-first spine's ``geographic_expand`` stage (microcosm#931) gives + every household the index of its geographic clone in + ``household_clone_index``, and its copies keep their root's; a household + table without the column is one clone. Refuses a non-integer column, a household without an index and a negative index, each by count or dtype, never by id or value. """ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index d1da7420f9..0a1840abff 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -235,6 +235,11 @@ *FRS_HMRC_RETAINED_LEAF_COLUMNS, EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN, ) +#: The support channel's own columns. The households' source columns +#: (source_household_id, source_year, source_household_key) come from +#: geographic_expand, which writes them on every household of the +#: clone-first spine before this stage runs, so the stage reads them and its +#: copies inherit them; it no longer declares them (microcosm#931). UK_SPI_SUPPORT_CHANNEL_OUTPUT_COLUMNS = ( HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, support_channel_column("person"), @@ -246,9 +251,6 @@ support_channel_column("household"), support_clone_index_column("household"), support_source_id_column("household"), - "source_household_id", - "source_year", - "source_household_key", ) UK_SPI_INCOME_SPINE_OUTPUT_COLUMNS = ( "gift_aid", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py index 7eef85f911..1a92a97ef7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py @@ -23,6 +23,9 @@ stable_identity_uniforms, ) from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime.geographic_clones import ( + UK_GEOGRAPHIC_SOURCE_COLUMNS, +) from microcosm.build.uk_runtime.hmrc_income import ( HMRC_SPI_ASSESSABLE_INCOME_COLUMN, ) @@ -933,7 +936,33 @@ def _prepare_household_lineage( *, source_year: int | None, ) -> pd.DataFrame: + """The households with their source columns. + + On the clone-first spine, geographic_expand writes every household's + source columns before this stage runs, and a clone's own ids are not its + source, so a household table carrying the geographic clone index must + carry all three and is refused, by column name alone, if it lacks any + (microcosm#931). A table without the clone index (the replacement path's + certified bases and unit fixtures) defaults the missing ones from its own + ids, as before: each household its own source. + """ + frame = household.copy() + if SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN in frame.columns: + missing = [ + column + for column in UK_GEOGRAPHIC_SOURCE_COLUMNS + if column not in frame.columns + ] + if missing: + raise ValueError( + "The SPI support channel's households carry " + f"{SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN!r} but not {missing}: on the " + "clone-first spine geographic_expand writes every household's " + "source columns, and the stage never takes a clone's own ids " + "for its source (microcosm#931)." + ) + return frame if "source_household_id" not in frame.columns: frame["source_household_id"] = frame["household_id"] if "source_year" not in frame.columns and source_year is not None: @@ -1093,13 +1122,14 @@ def _allocate_spi_prior_mass( """Give the SPI channel its share of every stratum's base mass. Where the household table carries ``household_clone_index`` (the - clone-first spine's geographic clones, microcosm#931), each clone is its - own stratum within ``strata_columns``, so every clone keeps its own mass - and a clone's SPI weights depend on its own base households and copies - alone; the stratum is derived for the allocation only and never written - to the frame. A household table without the column, as on every national - spine today, allocates exactly as before. A positive-mass stratum with no - SPI copy is refused, by count. + clone-first spine's geographic clones, which its ``geographic_expand`` + stage writes, microcosm#931), each clone is its own stratum within + ``strata_columns``, so every clone keeps its own mass and a clone's SPI + weights depend on its own base households and copies alone; the stratum + is derived for the allocation only and never written to the frame. A + household table without the column allocates exactly as before, and so + does a one-clone spine, whose clone stratum is the same for every + household. A positive-mass stratum with no SPI copy is refused, by count. """ household = result.household.copy() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index d12f82f81e..7b9301e943 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -120,10 +120,17 @@ ) from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.frs_spine import ( + FRS_AREA_CONDITIONING_COLUMNS, UKFRSSpineStageTransform, uk_frs_spine_seed_frame, ) from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform +from microcosm.build.uk_runtime.geographic_expand import ( + UKGeographicExpandStageTransform, +) +from microcosm.build.uk_runtime.geographic_identity import ( + UKGeographicIdentityStageTransform, +) from microcosm.build.uk_runtime.graph import ( UK_SPINE_ASSEMBLED_FINAL_STAGE, uk_registry, @@ -248,6 +255,21 @@ ) _RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" _RUNG_ABORT_EXIT_CODE = 3 +#: The spine's geographic clone count K (microcosm#931): one, until the spine +#: build takes a count. The graph's geographic_expand node, the transform it +#: runs and the engine all take it, and the node refuses a transform that +#: makes another number of clones. +_GEOGRAPHIC_CLONES = 1 +#: Declared outputs whose nonzero share no stage record reports: the root's +#: area-conditioning carriers (frs_spine.FRS_AREA_CONDITIONING_COLUMNS, +#: microcosm#931), which stay out of every receipt and of the sidecar's +#: shares, whichever route built the records. +_UNREPORTED_SHARE_COLUMNS = frozenset(FRS_AREA_CONDITIONING_COLUMNS) +#: The library whose version the sidecar records beside ``engine_batches``: +#: where the engine computes the frame clone by clone, private-school +#: attendance stores its own restatement of the engine's percentile rank, by +#: microdf's weighted percentile rank (microcosm#931). +_BATCHED_RESTATEMENT_DEPENDENCY = "microdf-python" def _uk_spine_stage_names(spec) -> tuple[str, ...]: @@ -739,7 +761,11 @@ def _entity_row_counts(frame) -> dict[str, int]: return {entity: int(len(frame.table(entity))) for entity in frame.entities} -def _rules_engine() -> PolicyEngineUKEngine: +def _rules_engine(*, clones: int | None = None) -> PolicyEngineUKEngine: + """The pinned engine; ``clones``, the spine's clone count where the caller + knows it, makes the engine refuse a frame of any other clone count, and + ``None`` refuses none (microcosm#931).""" + try: import policyengine_uk # noqa: F401 except ImportError as exc: @@ -753,25 +779,35 @@ def _rules_engine() -> PolicyEngineUKEngine: assert_installed_engine_is_pinned() # Whether a stage's call runs clone by clone is decided from its frame, - # never set here (microcosm#931), so nothing about batching enters the - # engine identity hashed from _rules_engine_provenance. - return PolicyEngineUKEngine() + # never set here, and the clone count only refuses (microcosm#931), so + # nothing about batching enters the engine identity hashed from + # _rules_engine_provenance. + return PolicyEngineUKEngine(clones=clones) def _rules_engine_record(engine: object, frame) -> dict[str, object]: """The sidecar's ``rules_engine``: the engine's provenance and, only when the spine's final frame carries more than one clone, how the adapter runs a call over it clone by clone (``engine_batches``, its - :meth:`batch_record`). Read from the final frame, never from the calls - this process ran: a resumed spine's stages that hit their checkpoints - run no call, and it records what a fresh run does. The engine identity is - hashed from the provenance alone, so it does not move with batching.""" + :meth:`batch_record`) and, beside it, the version of microdf-python + (``microdf_python``), whose weighted percentile rank private-school + attendance then stores in place of the engine's ("not-installed" when + absent, as the graph binds it; microcosm#931). Read from the final frame, + never from the calls this process ran: a resumed spine's stages that hit + their checkpoints run no call, and it records what a fresh run does. The + engine identity is hashed from the provenance alone, so it does not move + with batching.""" record = _rules_engine_provenance() batch_record = getattr(engine, "batch_record", None) batches = batch_record(frame) if callable(batch_record) else None if batches is not None: record["engine_batches"] = dict(batches) + try: + microdf = metadata.version(_BATCHED_RESTATEMENT_DEPENDENCY) + except metadata.PackageNotFoundError: + microdf = "not-installed" + record["microdf_python"] = microdf return record @@ -970,7 +1006,7 @@ def _build_sidecar( "stage": record.stage, "produced": list(record.produced), "rewrote": list(record.rewrote), - "nonzero_share": dict(record.nonzero_share), + "nonzero_share": _reported_shares(record.nonzero_share), "seconds": record.seconds, } for record in records @@ -1028,6 +1064,16 @@ def _copy_spine_checkpoint(output: Path, checkpoint: Path) -> None: temporary_path.unlink(missing_ok=True) +def _reported_shares(shares: Mapping[str, float]) -> dict[str, float]: + """A stage record's nonzero shares without the unreported columns.""" + + return { + column: share + for column, share in shares.items() + if column not in _UNREPORTED_SHARE_COLUMNS + } + + def _nonzero_shares(frame, columns: list[str]) -> dict[str, float]: shares: dict[str, float] = {} for column in columns: @@ -1063,7 +1109,8 @@ def _graph_stage_records( so the root node exposes no artifact for them although ``frs_spine`` declares them as outputs. Their share is read from the final population instead, which is what the legacy plan recorded (identity columns are - never zero, so the value is 1.0 on every vintage). + never zero, so the value is 1.0 on every vintage). The root's + area-conditioning carriers get no share (``_UNREPORTED_SHARE_COLUMNS``). """ structural = _structural_columns(frame) @@ -1079,6 +1126,8 @@ def _graph_stage_records( # A declared rewrite is an owned cell of its node, as an output is, so # its share reads the node's artifact too (microcosm#721). for column in (*stage.outputs, *stage.rewrites): + if column in _UNREPORTED_SHARE_COLUMNS: + continue matches = [ (coordinate, key) for coordinate, key in output_receipt.artifacts.items() @@ -1548,6 +1597,7 @@ def prepare_uk_spine_execution( source_mode="split", sample_fraction=args.sample_fraction, sample_seed=args.sample_seed, + geographic_clones=_GEOGRAPHIC_CLONES, ) stage_names = _uk_spine_stage_names(spec) if ( @@ -1626,7 +1676,7 @@ def prepare_uk_spine_execution( raise ValueError("--aps-tab is required when aps_occupation is scheduled.") stages = [stages_by_name[name] for name in stage_names] synthetic_fixture = _synthetic_fixture_evidence(args.synthetic_fixture_dir) - engine = _rules_engine() + engine = _rules_engine(clones=_GEOGRAPHIC_CLONES) stochastic_contract = load_uk_take_up_contract() frs_release = load_uk_frs_release() hmrc_spine_transform = _GraphSourceTransform( @@ -1809,6 +1859,15 @@ def prepare_uk_spine_execution( stage=stages_by_name["frs_annual_income_spells"], ) ) + if "geographic_expand" in stage_names: + implementations["geographic_expand"] = UKGeographicExpandStageTransform( + stage=stages_by_name["geographic_expand"], + clones=_GEOGRAPHIC_CLONES, + ) + if "geographic_identity" in stage_names: + implementations["geographic_identity"] = UKGeographicIdentityStageTransform( + stage=stages_by_name["geographic_identity"] + ) implementations["spi_support_channel"] = UKSPISupportChannelStageTransform( stage=stages_by_name["spi_support_channel"], sample_fraction=args.sample_fraction, @@ -2434,13 +2493,15 @@ def _checkpoint_link(path: Path) -> str: column for record in records for column in (*record.produced, *record.rewrote) + if column not in _UNREPORTED_SHARE_COLUMNS ) ) atomic_write_json( args.emit_nonzero_shares, { "stages": { - record.stage: dict(record.nonzero_share) for record in records + record.stage: _reported_shares(record.nonzero_share) + for record in records }, "final": _nonzero_shares(frame, final_columns), }, diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine/uk/test_uk_graph.py index a3d5f4030a..d7b69e6b98 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_graph.py @@ -78,15 +78,19 @@ def test_driver_projects_a_stage_record_for_every_graph_stage_on_the_fixture( manifest=manifest, store=store, stages=stages, frame=final ) + from microcosm.build.uk_runtime.frs_spine import FRS_AREA_CONDITIONING_COLUMNS + assert [record.stage for record in records] == [stage.stage for stage in stages] by_stage = {record.stage: record for record in records} for stage in stages: - # A declared rewrite is recorded beside the outputs (microcosm#721). + # A declared rewrite is recorded beside the outputs (microcosm#721); + # the root's area-conditioning carriers get no share (microcosm#931). assert set(by_stage[stage.stage].nonzero_share) == { *stage.outputs, *stage.rewrites, - }, stage.stage + } - set(FRS_AREA_CONDITIONING_COLUMNS), stage.stage assert by_stage[stage.stage].rewrote == stage.rewrites, stage.stage + assert set(FRS_AREA_CONDITIONING_COLUMNS) <= set(by_stage["frs_spine"].produced) root_shares = by_stage["frs_spine"].nonzero_share for column in ( "person_id", diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py index 8dcd55b2b5..4c22a42f5b 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_release_input_coverage.py @@ -508,6 +508,7 @@ def test_shipped_manifest_is_current(self) -> None: { "frs_hmrc_spine_leaves", "frs_annual_income_spells", + "geographic_expand", "spi_support_channel", "spi_income_band_donors", "hmrc_spi_income_spine", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py index f395a49891..b5fc46d1ae 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py @@ -1216,8 +1216,10 @@ def test_uk_source_manifest_contains_only_canonical_spine_stages(self) -> None: # hmrc_spi_income_spine (PolicyEngine/uk-autumn-budget-dashboard-2026#316); # microcosm#1490 spi_employment_shell after it and aps_occupation next; # microcosm#1095 (P1) pip_points just before trading_allowance_receipts; - # microcosm#1095 (Y1) uc_limited_capability just before uc_take_up. - assert len(spec.sources.stages) == 56 + # microcosm#1095 (Y1) uc_limited_capability just before uc_take_up; + # microcosm#931 geographic_expand and geographic_identity after + # frs_annual_income_spells, before frs_take_up. + assert len(spec.sources.stages) == 58 assert not {"frs_hmrc_retained_leaves", "hmrc_spi_income"}.intersection( stage.stage for stage in spec.sources.stages ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 57cc32d87c..1358bc1ee9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -2360,9 +2360,10 @@ def test_operation_dictionary_restates_the_reviewed_design(): "maximum_copy_weight), the quotient snapped to the nearest integer " "within clone-count ulps when the clone count exceeds one; every copy " "carries weight / copies, except where none of the 64 heaviest " - "households left whole can carry the exact-total correction, when the " - "shared fallback can put it on a root or copy, off weight / copies by " - "about one ulp of the total weight; of those 64, a selected household " + "households left whole, tried heaviest first, tied weights in row " + "order, can carry the exact-total correction, when the shared fallback " + "can put it on a root or copy, off weight / copies by about one ulp of " + "the total weight; of those 64, a selected household " "left at one copy can carry the correction only where its corrected " "weight keeps its one copy and the walk's selection, a check the " "fallback does not make" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py index 32c03a8b32..a14e867759 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py @@ -3,10 +3,9 @@ Invented households go through the real geographic expansion (G1, ``geographic_expand``), then the SPI support channel (stage 16), the SPI income band donors (stage 17) and the CGT support split (stage 42), each run by its -stage transform under the packaged stage declaration. G1's stage is a stand-in -until the spine declares it. The frame runs at K = 1, 2, 3, 7 and 15 -geographic clones; it is small, so K = 15 takes well under a second. Each K is -held to K = 1: +stage transform under the packaged stage declaration, G1's included. The +frame runs at K = 1, 2, 3, 7 and 15 geographic clones; it is small, so K = 15 +takes well under a second. Each K is held to K = 1: - **Reported first:** stage 17's mass scale at K = 1 and at each K. Clone 0's stage-17 weights nest on K = 1's only while the scale is the one a one-clone @@ -41,6 +40,14 @@ copy counts. - **The stand-in:** G1's output is ``clone_first_frame``'s on the stand-in's columns, ids and weights, so the tests written on the stand-in hold for G1. +- **The draws behind the clones:** frs_take_up, frs_person_draws and + frs_household_draws run on every clone, after G1. On the stand-in at K = 3, + each take-up flag's population support and the apprentice draw's + population and eligible supports are K = 1's, since the stages count + support in true source households. Shares, and counts that follow a draw + (the apprentices flagged, how many values a flag takes), are left out, and + the extended hours are certain, so the under-two usage population follows + no draw. Invented values only. Assertion messages give K, stages, counts and booleans, never a weight or an id. @@ -61,7 +68,12 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.uk_runtime import cgt_support, spi_band_donors, spi_support +from microcosm.build.uk_runtime import ( + cgt_support, + frs_person_draws, + spi_band_donors, + spi_support, +) from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_INVESTABLE_WEALTH_COLUMNS, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, @@ -75,6 +87,11 @@ UKCGTSupportSplitStageTransform, cgt_support_income_band, ) +from microcosm.build.uk_runtime.frs_household_draws import ( + UKFRSHouseholdDrawsStageTransform, +) +from microcosm.build.uk_runtime.frs_person_draws import UKFRSPersonDrawsStageTransform +from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.geographic_expand import ( GEOGRAPHIC_EXPAND_STAGE_NAME, @@ -86,6 +103,10 @@ UK_NATIONAL_SCHEMA, uk_national_frame, ) +from microcosm.build.uk_runtime.receipt_counts import ( + UK_BELOW_TEN_BAND, + support_band, +) from microcosm.build.uk_runtime.spi_band_donors import ( HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, PERSON_IS_SPI_INCOME_BAND_CARRIER, @@ -123,6 +144,7 @@ base_id_tables, clone_first_frame, ) +from test_support.microcosm_build.uk_frs_stochastic import _POLICY, _Contract G1 = GEOGRAPHIC_EXPAND_STAGE_NAME SPI = UK_SPI_SUPPORT_STAGE_NAME @@ -280,20 +302,6 @@ def _age_model() -> SPIDonorAgeModel: ) -def _g1_stage() -> SourceStageSpec: - """A stand-in stage for G1: no stage declares it until the spine wires it.""" - - return SourceStageSpec( - stage=G1, - survey="Invented test stage", - source="Invented test stage", - grain="household", - artifacts=(), - operations=(), - outputs=UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, - ) - - def _packaged_stage(name: str) -> SourceStageSpec: spec = load_country_spec("uk") assert spec.sources is not None @@ -341,7 +349,7 @@ def recorded(values, *args, **kwargs): def _run(base: Frame, clones: int, handed: Mapping[str, list[np.ndarray]]) -> _Run: transforms = { - G1: UKGeographicExpandStageTransform(_g1_stage(), clones=clones), + G1: UKGeographicExpandStageTransform(_packaged_stage(G1), clones=clones), SPI: UKSPISupportChannelStageTransform( stage=_packaged_stage(SPI), sample_fraction=_SAMPLE_FRACTION, @@ -1070,3 +1078,167 @@ def test_rule_b_puts_the_correction_on_a_copy_at_seven_and_thirteen_clones( assert int(last[CGT_SUPPORT_COPY_INDEX_COLUMN]) == 126 # About an ulp of the total off w / n: below 1e-12 of the copy's weight. assert abs(weights[-1] - values[-1]) < 1e-12 * values[-1] + + +# --- the draws behind the clones ---------------------------------------------------- +# +# frs_take_up, frs_person_draws and frs_household_draws run on every clone, +# after G1, and count their receipts' support before the copies, in true +# source households (microcosm#931). Invented households: four families, each +# with a claimant in work, a child under two and a child of three in their +# unit, and an employee of an apprentice's age in a unit of their own; and +# four pensioners living alone. Half are in Scotland. + +_DRAW_FAMILIES = 4 +_DRAW_PENSIONERS = 4 +#: Receipt fields that are not counts: the flags' shares, and how many values +#: a flag takes, which grows where clones draw apart on their own ids. +_DRAW_SHARE_FIELDS = frozenset({"target", "weighted_share", "distinct_values"}) + + +class _CertainHoursContract(_Contract): + """test_support's take-up contract with the extended hours certain. + + The under-two usage flag's population is the children under two in units + drawn to claim the extended hours. Each clone draws its own claimants, on + its own ids, so below a rate of one that population, and its support, + grows with K; at one it is every such child on every clone.""" + + rates = {**_Contract.rates, "extended_childcare": 1.0} + + +def _draws_frame() -> Frame: + people: list[dict[str, object]] = [] + units: list[dict[str, object]] = [] + households: list[dict[str, object]] = [] + for sernum in range(1, _DRAW_FAMILIES + _DRAW_PENSIONERS + 1): + households.append( + { + "household_id": sernum, + "region": "SCOTLAND" if sernum % 2 else "LONDON", + "household_weight": 300.0 + 40.0 * sernum, + } + ) + # (unit, age, employment status, UC claimant or partner) + if sernum <= _DRAW_FAMILIES: + members = ( + (1, 38, "FT_EMPLOYED", True), + (1, 1, "CHILD", False), + (1, 3, "CHILD", False), + (2, 22, "FT_EMPLOYED", True), + ) + else: + members = ((1, 70, "RETIRED", True),) + units.extend( + {"benunit_id": sernum * 100 + unit, "is_married": False} + for unit in sorted({member[0] for member in members}) + ) + for position, (unit, age, status, claimant) in enumerate(members, start=1): + people.append( + { + "person_id": sernum * 1000 + position, + "person_benunit_id": sernum * 100 + unit, + "person_household_id": sernum, + "age": age, + "employment_status": status, + "months_since_last_birthday": 6.0, + "is_uc_claimant": claimant, + "child_benefit_reported": 0, + "pension_credit_reported": 0, + "universal_credit_reported": 0, + } + ) + return uk_national_frame( + person=pd.DataFrame(people), + benunit=pd.DataFrame(units), + household=pd.DataFrame(households), + time_period=_PERIOD, + ) + + +def _draw_counts(frame: Frame) -> dict[str, dict[str, object]]: + """frs_take_up, frs_person_draws and frs_household_draws on ``frame``, in + spine order, under their packaged declarations, and the count fields of + their receipts: each take-up flag's population support, with its nesting + count or why it is unmeasured, and the apprentice draw's population and + eligible supports. The flags' shares and the apprentices' masses and + flagged support are left out: the anchored residual and the apprentice + walk pool over the clones, whose draws are keyed on their own ids.""" + + contract = _CertainHoursContract() + transforms = ( + UKFRSTakeUpStageTransform( + contract=contract, + stage=_packaged_stage("frs_take_up"), + population_policy=_POLICY, + ), + UKFRSPersonDrawsStageTransform( + contract=contract, stage=_packaged_stage("frs_person_draws") + ), + UKFRSHouseholdDrawsStageTransform( + contract=contract, stage=_packaged_stage("frs_household_draws") + ), + ) + counts: dict[str, dict[str, object]] = {} + for transform in transforms: + frame = transform(frame) + evidence = transform.checkpoint_metadata()["evidence"] + counts[evidence["stage"]] = { + "flags": [ + { + key: value + for key, value in row.items() + if key not in _DRAW_SHARE_FIELDS + } + for row in evidence["take_up_signal"]["flags"] + ] + } + if "apprentice_receipt" in evidence: + receipt = evidence["apprentice_receipt"] + counts[evidence["stage"]]["apprentices"] = { + key: receipt[key] for key in ("population_support", "eligible_support") + } + return counts + + +def test_the_draws_count_in_source_households_on_every_clone(monkeypatch) -> None: + # The apprentices' relief reads an installed engine; their counts do not. + monkeypatch.setattr(frs_person_draws, "_installed_uk_system", lambda: None) + base = _draws_frame() + one = _draw_counts(clone_first_frame(base, 1)) + three = clone_first_frame(base, 3) + assert _draw_counts(three) == one + + # Not vacuous: every household flag rests on the eight households, which + # reads "3-9", and so do the apprentices, while three clones' rows would + # read another band. + households = len(base.table("household")) + assert { + row["population_support"] for row in one["frs_household_draws"]["flags"] + } == {UK_BELOW_TEN_BAND} + assert one["frs_person_draws"]["apprentices"] == { + "population_support": UK_BELOW_TEN_BAND, + "eligible_support": UK_BELOW_TEN_BAND, + } + assert support_band(3 * households) != UK_BELOW_TEN_BAND + # Without the lineage, as a node not given it would read the frame, the + # draws count each clone as a household of its own. + lineage = [SOURCE_HOUSEHOLD_ID_COLUMN, ladder_clone_index_column("household")] + rows = Frame( + { + entity: ( + three.table(entity).drop(columns=lineage) + if entity == "household" + else three.table(entity) + ) + for entity in three.entities + }, + three.schema, + {"household": three.weights_for("household")}, + three.strata, + metadata=three.metadata, + ) + assert { + row["population_support"] + for row in _draw_counts(rows)["frs_household_draws"]["flags"] + } == {support_band(3 * households)} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py index 2dec44d4ff..07db7b1f71 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -24,6 +24,7 @@ cgt_structure, cgt_support, copy_id_table, + geographic_expand, graph_kernels, high_value_home_support, ) @@ -82,8 +83,10 @@ "cgt_residential_split", ) #: The spine stage that mints each copy field (the band donors share the SPI -#: copies' field at ordinal 2). +#: copies' field at ordinal 2, and the geographic clones' field is minted by +#: the clone-first spine's geographic expansion, microcosm#931). _MINTING_STAGES = { + "geographic_support": "geographic_expand", "spi_support_channel": "spi_support_channel", "high_value_support": "high_value_home_support", "cgt_support_split": "cgt_support_split", @@ -139,6 +142,7 @@ def test_the_copy_fields_are_the_lineage_branches_in_stage_order() -> None: if name in _MINTING_STAGES ] assert order == sorted(order) + assert len(order) == len(_FIELD_NAMES) - 1 assert position["spi_income_band_donors"] < position["high_value_home_support"] @@ -183,6 +187,14 @@ def _declared_id_remapping(operations, kind: str) -> str: @pytest.mark.parametrize( ("field", "remapping", "ordinal_text"), [ + ( + geographic_expand.GEOGRAPHIC_COPY_ID_FIELD, + _declared_id_remapping( + geographic_expand.geographic_expand_operation_parameters(), + geographic_expand.GEOGRAPHIC_EXPAND_OPERATION_KIND, + ), + "k x the field's radix", + ), ("spi_support_channel", SPI_SUPPORT_ID_REMAPPING, "1 x the field's radix"), ( "spi_support_channel", @@ -390,7 +402,9 @@ def _spi_support_on_clone_first_tables(base: IdTables, n_clones: int): """The real SPI support stage on ``n_clones`` geographic clones of ``base``. Every clone of the same base households is selected, so only the stage's - id minting can tell K=1 from K=3 apart. + id minting can tell K=1 from K=3 apart. The households carry the source + columns the geographic expansion writes, which the stage refuses a + clone-indexed table without (microcosm#931). """ tables = clone_first_tables(base, n_clones) @@ -399,12 +413,18 @@ def _spi_support_on_clone_first_tables(base: IdTables, n_clones: int): for source in base.household["household_id"].tolist() if source % 3 != 0 ] + household = tables.household[ + ["household_id", "source_household_id", "household_clone_index"] + ].assign(household_weight=1.0, source_year=np.int64(2024)) + household["source_household_key"] = pd.Series( + [f"2024:{source}" for source in household["source_household_id"].tolist()], + index=household.index, + dtype=object, + ) return create_uk_spi_support_tables( person=tables.person, benunit=tables.benunit, - household=tables.household[ - ["household_id", "source_household_id", "household_clone_index"] - ].assign(household_weight=1.0), + household=household, expected_spi_households=None, selected_household_ids=tuple( source + clone * GEOGRAPHIC_RADIX @@ -515,8 +535,37 @@ def test_the_executor_takes_the_declared_radix_where_the_frame_maximum_differs( np.testing.assert_array_equal(lineage.to_numpy(), parent(targets, field)) +@pytest.mark.parametrize("clones", [1, 3]) +def test_the_executor_traces_each_geographic_clone_to_its_clone_zero_row( + clones: int, +) -> None: + """The geographic expansion's clones, as the executor reads them: clone k + of a row is the row plus k x the geographic_support field's radix, so + every new row's parent is its clone-0 row, and one clone adds no row + (microcosm#931).""" + + base = base_id_tables() + before = id_frame(base) + after = id_frame(clone_first_tables(base, clones)) + offset = graph_kernels._declared_copy_radix("geographic_expand", before) + assert offset == radix("geographic_support") + for entity in before.entities: + lineage = graph_kernels._source_lineage(before, after, entity, id_offset=offset) + targets = lineage.index.to_numpy(dtype=np.int64) + assert len(targets) == (clones - 1) * len(before.table(entity)) + assert lineage.dtype == np.dtype("int64") + np.testing.assert_array_equal( + lineage.to_numpy(), parent(targets, "geographic_support") + ) + np.testing.assert_array_equal( + ordinal(targets, "geographic_support"), + np.repeat(np.arange(1, clones), len(before.table(entity))), + ) + + def test_the_executor_names_each_stages_own_copy_id_field() -> None: assert dict(graph_kernels._COPY_ID_FIELDS) == { + "geographic_expand": geographic_expand.GEOGRAPHIC_COPY_ID_FIELD, "high_value_home_support": high_value_home_support.HIGH_VALUE_SUPPORT_COPY_ID_FIELD, "cgt_support_split": cgt_support.CGT_SUPPORT_COPY_ID_FIELD, "cgt_incidence_clone": cgt_structure.CGT_CLONE_COPY_ID_FIELD, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index b69bf48539..f430f21b89 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -48,6 +48,7 @@ BENEFIT_CODES, BENEFIT_RECIPIENT_RULE, BSP_MONTHLY_RATES, + FRS_AREA_CONDITIONING_COLUMNS, FRS_CARE_HELPER_BIT, FRS_CARE_OUTSIDE_BIT, FRS_CARE_RECIPIENTS_COLUMN, @@ -58,9 +59,15 @@ FRS_COMPANY_CAR_PRICE_BAND_BOUNDS, FRS_COMPANY_CAR_PRICE_BAND_OPEN, FRS_COMPANY_CAR_PRICE_BANDS, + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, + FRS_IMD_DECILE_COLUMN, + FRS_IMD_DECILE_COLUMN_BY_NATION, + FRS_IMD_DECILE_UNOBSERVED, + FRS_IMD_DECILES, FRS_SPINE_TABLES, FRS_UC_CARER_ELEMENT_COLUMN, FRS_UC_LCW_STATUS_COLUMN, + OUTPUT_COLUMNS, OWNER_MANAGER_COLUMNS, REGION_MAP, SCOTTISH_WATER_CHARGES_REDUCTION_RECIPIENT_SHARE, @@ -76,7 +83,9 @@ frs_care_recipients, frs_claimant_or_partner, frs_council_tax_band, + frs_council_tax_band_observed, frs_dwp_carer_care_hours, + frs_imd_decile, frs_lcwra_proxy, frs_uc_carer_element, frs_uc_lcw_status, @@ -252,6 +261,12 @@ def _fixture_tables() -> dict[str, list[dict[str, object]]]: "INTDATE": float(_FIXTURE_INTERVIEW_SAS_DATE), # A conventional household (HHSTAT 1), not a shared one. "HHSTAT": 1, + # Each nation's IMD decile variable; a household has its own nation's + # (England here) and the others blank (microcosm#931). + "IMDE": 3, + "IMDW": "", + "IMDS": "", + "IMDN": "", } household_1 = { **household_2, @@ -262,6 +277,8 @@ def _fixture_tables() -> dict[str, list[dict[str, object]]]: "PTENTYP2": 6, "TYPEACC": 4, "BEDROOM6": 2, + "IMDE": "", + "IMDS": 7, "CTANNUAL": -1.0, "CTBAND": 2, "CTDISC": 1, @@ -706,6 +723,10 @@ def _manifest_stage() -> SourceStageSpec: def _synthetic_spec(stage: SourceStageSpec) -> SimpleNamespace: + packaged = load_country_spec("uk") + assert packaged.sources is not None + packaged_stages = packaged.sources.stage_map() + def source_stage( name: str, *, @@ -889,6 +910,31 @@ def source_stage( "disabled_students_allowance_eligible_expenses", ), ), + # microcosm#931: the HMRC leaves read the raw tabs, so they + # run before the geographic clones, which the draws follow, as + # in the packaged roster; the clones and their keys run under + # their packaged declarations. + source_stage( + "frs_hmrc_spine_leaves", + tables=("adult", "benefits"), + operations=[ + {"kind": "retain_adjudicated_frs_hmrc_leaves"}, + { + "kind": "derive", + "output": "employer_pension_contributions", + }, + ], + outputs=( + "hmrc_spi_pay", + "hmrc_spi_unemployment_benefit_income", + "hmrc_spi_incapacity_benefit_income", + "ossben_identifiable_subset", + "srp_regular_code5", + "employer_pension_contributions", + ), + ), + packaged_stages["geographic_expand"], + packaged_stages["geographic_identity"], source_stage( "frs_take_up", grain="benunit", @@ -1064,25 +1110,6 @@ def source_stage( }, ), ), - source_stage( - "frs_hmrc_spine_leaves", - tables=("adult", "benefits"), - operations=[ - {"kind": "retain_adjudicated_frs_hmrc_leaves"}, - { - "kind": "derive", - "output": "employer_pension_contributions", - }, - ], - outputs=( - "hmrc_spi_pay", - "hmrc_spi_unemployment_benefit_income", - "hmrc_spi_incapacity_benefit_income", - "ossben_identifiable_subset", - "srp_regular_code5", - "employer_pension_contributions", - ), - ), source_stage( "spi_support_channel", grain="household", @@ -1125,9 +1152,6 @@ def source_stage( "household_support_channel", "household_support_clone_index", "household_source_id", - "source_household_id", - "source_year", - "source_household_key", ), ), source_stage( @@ -1168,6 +1192,8 @@ def source_stage( geography_spine=None, resource_hashes={ "brma_private_rented_households.json": "f" * 64, + # The geography keys' FRS release, the packaged bytes'. + "frs_release.json": packaged.resource_hashes["frs_release.json"], # S2(a): frs_council_tax levels the Scottish bills to these facts. "scotgov_band_d_liability_facts.json": "f" * 64, }, @@ -1190,6 +1216,13 @@ def materialize(self, frame, variables, period): return values +def _fake_rules_engine(*, clones: int | None = None) -> _FakeUKEngine: + """The driver's engine, as the spine build asks for it: with the spine's + clone count, which the stand-in does not check (microcosm#931).""" + + return _FakeUKEngine() + + def test_manifest_stage_and_runtime_agree_on_artifacts_and_operations() -> None: stage = _manifest_stage() @@ -2858,6 +2891,169 @@ def test_northern_ireland_households_take_the_placeholder_band() -> None: assert frs_council_tax_band(household).tolist() == ["D", "D", "E", "I", "D"] +def test_the_root_writes_each_households_imd_decile_and_band_flag_typed( + tmp_path: Path, +) -> None: + """The root's two area-conditioning carriers (microcosm#931): the decile + int64, the band flag bool, both declared outputs of the stage. The + Scottish fixture household reads IMDS and the English one IMDE.""" + + stage = _write_fixture(tmp_path) + + household = ( + build_uk_frs_spine_frame(tmp_path, stage=stage) + .table("household") + .set_index("household_id") + ) + + assert household[FRS_IMD_DECILE_COLUMN].dtype == np.dtype("int64") + assert household[FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN].dtype == np.dtype(bool) + assert household[FRS_IMD_DECILE_COLUMN].to_dict() == {1: 7, 2: 3} + assert household[FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN].to_dict() == { + 1: True, + 2: True, + } + assert FRS_AREA_CONDITIONING_COLUMNS == ( + FRS_IMD_DECILE_COLUMN, + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, + ) + assert set(FRS_AREA_CONDITIONING_COLUMNS) <= set(OUTPUT_COLUMNS) + assert set(FRS_AREA_CONDITIONING_COLUMNS) <= set(_manifest_stage().outputs) + + +def test_the_root_names_the_conditioned_supports_decile_and_every_nation() -> None: + """The decile's name is spelled in frs_spine, so a change to the + conditioned supports' module never re-keys the root; the two spellings, + the decile codes and the nations stay equal (microcosm#931).""" + + from microcosm.build.uk_runtime.atomic_area_support import ( + IMD_DECILE_COLUMN, + IMD_DECILES, + ) + from microcosm.build.uk_runtime.council_tax_constants import UK_NATION_BY_REGION + + assert FRS_IMD_DECILE_COLUMN == IMD_DECILE_COLUMN + assert tuple(str(decile) for decile in FRS_IMD_DECILES) == IMD_DECILES + assert FRS_IMD_DECILE_UNOBSERVED not in FRS_IMD_DECILES + assert set(FRS_IMD_DECILE_COLUMN_BY_NATION) == set(UK_NATION_BY_REGION.values()) + assert sorted(FRS_IMD_DECILE_COLUMN_BY_NATION.values()) == [ + "imde", + "imdn", + "imds", + "imdw", + ] + + +def test_imd_decile_reads_the_households_own_nation_and_codes_one_to_ten() -> None: + # England, Wales, Scotland and Northern Ireland read their own variables. + # Then 0 for: a blank region code, codes 0 and 11, a non-integral code, a + # blank own variable beside another nation's code, and a negative code. + household = pd.DataFrame( + { + "gvtregno": [1, 11, 12, 13, np.nan, 8, 9, 10, 5, 4], + "imde": [1, 9, 9, 9, 5, 0, 11, 5.5, np.nan, -1], + "imdw": [9, 10, 9, 9, 5, 9, 9, 9, 9, 9], + "imds": [9, 9, 2, 9, 5, 9, 9, 9, 9, 9], + "imdn": [9, 9, 9, 3, 5, 9, 9, 9, 9, 9], + }, + index=range(101, 111), + ) + + decile = frs_imd_decile(household) + + assert decile.dtype == np.dtype("int64") + assert decile.index.equals(household.index) + assert decile.tolist() == [1, 10, 2, 3, 0, 0, 0, 0, 0, 0] + + +def test_the_band_flag_is_false_unless_the_frs_gives_a_band_its_region_uses() -> None: + # Northern Ireland pays domestic rates; codes outside 1-9 and blanks take + # the placeholder; band I is Welsh only; a household of no known region + # holds no band of its region. council_tax_band itself is unchanged. + household = pd.DataFrame( + { + "gvtregno": [13, 13, 6, 11, 6, 12, 1, np.nan, 12, 8], + "ctband": [5, 0, 5, 9, 0, 9, 9, 4, 8, np.nan], + }, + index=range(201, 211), + ) + + observed = frs_council_tax_band_observed(household) + + assert frs_council_tax_band(household).tolist() == [ + *("D", "D", "E", "I", "D"), + *("I", "I", "D", "H", "D"), + ] + assert observed.dtype == np.dtype(bool) + assert observed.index.equals(household.index) + assert observed.tolist() == [ + *(False, False, True, True, False), + *(False, False, False, True, False), + ] + + +@pytest.mark.parametrize("column", ["IMDE", "IMDW", "IMDS", "IMDN"]) +def test_frs_spine_refuses_a_household_tab_without_an_imd_variable( + tmp_path: Path, column: str +) -> None: + """A vintage without one of the four IMD variables refuses, naming it and + no count; it never reads the absent variable as zeros (microcosm#931).""" + + tables = _fixture_tables() + tables["househol"] = [ + {key: value for key, value in row.items() if key != column} + for row in tables["househol"] + ] + stage = _write_fixture(tmp_path, tables) + + with pytest.raises(KeyError) as caught: + build_uk_frs_spine_frame(tmp_path, stage=stage) + assert caught.value.args == ( + f"The FRS household tab has no {[column]} column(s), so the IMD decile " + "cannot be set.", + ) + + +def test_the_imd_and_band_codes_move_only_the_two_carriers(tmp_path: Path) -> None: + """Every other column of the root is the same whatever the IMD codes say, + and a code outside 1-9 moves only the band flag beside the placeholder + band it already took (microcosm#931).""" + + tables = _fixture_tables() + before = build_uk_frs_spine_frame( + tmp_path / "before", stage=_write_fixture(tmp_path / "before", tables) + ) + for row in tables["househol"]: + row.update({"IMDE": 10, "IMDW": 10, "IMDS": 10, "IMDN": 10}) + if row["SERNUM"] == 2: + # Band D either way: code 4 is band D, code 0 the placeholder. + row["CTBAND"] = 0 + after = build_uk_frs_spine_frame( + tmp_path / "after", stage=_write_fixture(tmp_path / "after", tables) + ) + + moved = { + entity: [ + column + for column in before.table(entity).columns + if not before.table(entity)[column].equals(after.table(entity)[column]) + ] + for entity in before.entities + } + assert moved == { + "person": [], + "benunit": [], + "household": [FRS_IMD_DECILE_COLUMN, FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN], + } + household = after.table("household").set_index("household_id") + assert household[FRS_IMD_DECILE_COLUMN].to_dict() == {1: 10, 2: 10} + assert household[FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN].to_dict() == { + 1: True, + 2: False, + } + assert household["council_tax_band"].to_dict() == {1: "B", 2: "D"} + + def test_household_and_benunit_mapping_values_are_ported(tmp_path: Path) -> None: stage = _write_fixture(tmp_path) @@ -3133,7 +3329,14 @@ def test_driver_writes_spine_h5_sidecars_and_logbook( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + # The engine the build asks for, by the clone count it asks with. + asked: list[int | None] = [] + + def recording_rules_engine(*, clones: int | None = None) -> _FakeUKEngine: + asked.append(clones) + return _fake_rules_engine(clones=clones) + + monkeypatch.setattr(tool, "_rules_engine", recording_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) monkeypatch.delenv("POPULACE_LEDGER_URL", raising=False) @@ -3195,6 +3398,18 @@ def test_driver_writes_spine_h5_sidecars_and_logbook( "property_purchased", "brma", } <= set(frame.table("household")) + # The spine build clones every household once (microcosm#931): it asks + # for its engine at that one clone, so the adapter refuses a frame of any + # other clone count; every clone index is zero, each household is its own + # source, and each has a geography key of its own. + assert asked == [1] + household = frame.table("household") + for entity in ("person", "benunit", "household"): + assert set(frame.table(entity)[f"{entity}_clone_index"].tolist()) == {0} + assert household["source_household_id"].tolist() == ( + household["household_id"].tolist() + ) + assert household["geography_household_key"].is_unique sidecar = json.loads(output.with_suffix(".build.json").read_text()) assert sidecar["pipeline"] == "uk-frs-spine" # The build record says which of its stages read the SPI Public Use Tape @@ -3266,8 +3481,10 @@ def test_driver_writes_spine_h5_sidecars_and_logbook( "sha256": hashlib.sha256(replay_bytes).hexdigest(), } assert len(sidecar["stochastic_contract_sha256"]) == 64 + # The geography keys' stage declares the FRS release (microcosm#931). assert sidecar["resource_pins"] == { "brma_private_rented_households.json": "f" * 64, + "frs_release.json": load_country_spec("uk").resource_hashes["frs_release.json"], "scotgov_band_d_liability_facts.json": "f" * 64, } # Resolve the expected version the way the driver does, so the assertion @@ -3310,7 +3527,7 @@ def test_driver_kernels_free_each_stage_result_once_its_evidence_is_captured( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) @@ -3371,7 +3588,7 @@ def test_driver_writes_payload_identical_h5s( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) monkeypatch.delenv("POPULACE_LEDGER_URL", raising=False) @@ -3436,7 +3653,7 @@ def test_driver_keeps_only_the_final_graph_version( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) runs = [] @@ -3842,7 +4059,7 @@ def test_driver_records_sampled_spine_sidecar( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) # The root's rung sampler lives in national_sampling (microcosm#931). @@ -3909,7 +4126,7 @@ def test_driver_marks_full_fixture_smoke_outputs_non_release( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) assert ( @@ -4000,7 +4217,7 @@ def _run_recorded_smoke_spine( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) checkpoint_root = ( @@ -4160,7 +4377,7 @@ def test_driver_records_sanitized_failed_staging_lifecycle( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) secret = "operator-secret-value" @@ -4225,7 +4442,7 @@ def test_driver_records_an_operator_stop_as_a_discarded_attempt( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) @@ -4302,7 +4519,7 @@ def test_driver_materializes_a_blocked_assembled_gate_report_before_failing( # arms the spine battery and the graph gains its gate nodes. spec.gates = load_country_spec("uk").gates monkeypatch.setattr(tool, "load_country_spec", lambda country: spec) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) @@ -4400,7 +4617,7 @@ def test_driver_sampled_named_edge_aborts_with_receipt( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) @@ -4758,6 +4975,82 @@ def checkpoint_metadata(self) -> dict[str, object]: assert "frs_spine" not in evidence +def test_the_sidecar_reports_no_share_of_the_root_area_carriers() -> None: + """The root's area-conditioning carriers stay out of every stage record's + nonzero shares in the sidecar, whichever route built the records, and the + graph route's projection asks no artifact for them (microcosm#931).""" + + from microcosm.build.plan import StageRecord + + frame = uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 2], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [1, 2], + "region": ["LONDON", "WALES"], + FRS_IMD_DECILE_COLUMN: [0, 4], + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN: [False, True], + } + ), + time_period="2024", + household_weights=np.asarray([1.0, 2.0]), + ) + produced = ("household_id", "region", *FRS_AREA_CONDITIONING_COLUMNS) + # A record whose shares name the carriers, as a plan-built one does. + record = StageRecord( + stage="frs_spine", + produced=produced, + donor_survey=None, + nonzero_share={ + "household_id": 1.0, + "region": 1.0, + FRS_IMD_DECILE_COLUMN: 0.5, + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN: 0.5, + }, + ) + + sidecar = spine_build._build_sidecar( + frame=frame, + stages=[], + records=[record], + artifact_pins={}, + resource_pins={}, + input_artifact_pins={}, + hmrc_replay={}, + stochastic_contract_sha256="0" * 64, + frs_vintage="synthetic", + sampling=None, + ) + + (written,) = sidecar["stage_records"] + assert written["nonzero_share"] == {"household_id": 1.0, "region": 1.0} + assert written["produced"] == list(produced) + + # The graph route: the root's node exposes the region's artifact alone. + manifest = SimpleNamespace( + nodes={ + "frs_spine": SimpleNamespace(artifacts={("household", "region"): "k"}), + "create_uk_frs": SimpleNamespace(wall_time=1.5), + } + ) + store = SimpleNamespace(load_column=lambda key: pd.Series(["LONDON", "WALES"])) + stage = SimpleNamespace( + stage="frs_spine", outputs=produced, rewrites=(), survey="Synthetic FRS" + ) + (projected,) = spine_build._graph_stage_records( + manifest=manifest, store=store, stages=[stage], frame=frame + ) + assert projected.nonzero_share == {"household_id": 1.0, "region": 1.0} + assert projected.produced == produced + + def test_collect_fit_weight_records_is_duck_typed_and_fail_visible(): class _Record: def __init__(self, fit_name, weight_kind): @@ -4804,7 +5097,7 @@ def test_spine_completion_waits_for_attempt_record( monkeypatch.setattr( tool, "load_country_spec", lambda country: _synthetic_spec(stage) ) - monkeypatch.setattr(tool, "_rules_engine", lambda: _FakeUKEngine()) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) _stub_policy_readers(monkeypatch) spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) if late_error: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py index cb858e4f01..c85c34c14e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py @@ -164,6 +164,91 @@ def test_one_dense_clone_of_a_spine_at_the_top_copy_fields_expands_exactly( assert str(smallest) not in str(caught.value) +def _clone_first_source_frame(): + """The toy spine as a clone-first spine at one clone: the clone indices + and household source columns geographic_expand writes (microcosm#931).""" + frame = source_frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for entity, table in tables.items(): + table[ladder_clone_index_column(entity)] = np.zeros(len(table), dtype="int64") + household = tables["household"] + household["source_year"] = np.full(len(household), 2023, dtype="int64") + household["source_household_key"] = pd.array( + [f"2023:{source}" for source in household["source_household_id"]], + dtype="string", + ) + return type(frame)( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +@pytest.mark.parametrize("k", [1, 2]) +def test_dense_refuses_a_clone_first_spine_at_any_clone_count( + k, monkeypatch, toy_ladder, tmp_path +): + """uk.full.expand refuses a spine carrying geographic_expand's clone + index, at one dense clone as at more, naming the clone-first spine and + group (c), which builds the dense line on it; never the overflow refusal, + whose "n_clones=1 still expands" holds only without the index + (microcosm#931).""" + from microcosm.graph import NodeRejectedError + from test_support.microcosm_build import uk_full_population_graph + + _, path = toy_ladder + monkeypatch.setattr( + uk_full_population_graph, "source_frame", _clone_first_source_frame + ) + + with pytest.raises(NodeRejectedError) as caught: + _run_legacy(k, path, ContentStore(tmp_path / "refused")) + + message = str(caught.value) + assert ( + "input person table already carries the reserved in-memory clone column " + "'person_clone_index'; the ladder clone owns that name. A clone-first " + "spine, whose geographic_expand stage writes it, carries its geographic " + "clones already, and the dense line builds on such a spine only from " + "microcosm#931 group (c), at any n_clones." + ) in message + assert "overflow" not in message + + +def test_the_dense_lineage_defaults_only_without_the_clone_index() -> None: + """The dense expansion defaults a household's source columns from its own + ids only on a spine without the geographic clone index; on a clone-first + spine geographic_expand writes them, and a table carrying the index + without them is refused by column name (microcosm#931). The reserved + column refusal before it refuses such a table first.""" + from microcosm.build.uk_runtime.rowwise_dataset import _attach_source_lineage + + household = pd.DataFrame({"household_id": [1, 2]}) + defaulted = _attach_source_lineage(household.copy(), source_year=2023) + assert defaulted["source_household_id"].tolist() == [1, 2] + assert defaulted["source_year"].tolist() == [2023, 2023] + assert defaulted["source_household_key"].tolist() == ["2023:1", "2023:2"] + + cloned = household.assign(household_clone_index=[0, 1], source_household_id=[1, 1]) + with pytest.raises(ValueError) as caught: + _attach_source_lineage(cloned.copy(), source_year=2023) + assert str(caught.value) == ( + "The household table carries 'household_clone_index' but not " + "['source_year', 'source_household_key']: on the clone-first spine " + "geographic_expand writes every household's source columns, and the " + "dense line never takes a clone's own ids for its source (microcosm#931)." + ) + complete = cloned.assign( + source_year=[2023, 2023], source_household_key=["2023:1", "2023:1"] + ) + pd.testing.assert_frame_equal( + _attach_source_lineage(complete.copy(), source_year=2023), complete + ) + + def test_k_changes_expansion_but_never_sampling(): first, _ = graph_and_registry(1) second, _ = graph_and_registry(3) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py index e4caf6d841..2ef95e9716 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_target_graph.py @@ -28,6 +28,33 @@ def test_default_scope_contains_all_levels_and_never_depends_on_k_or_k_small(): assert graph.node("uk.full.target_selection").params["geography_levels"] is None +def test_the_identity_node_rewrites_the_key_the_clone_first_spine_carries(): + """The spine's geographic_identity stage keys every household + (microcosm#931), so the dense line's uk.full.identity declares its key as + a rewrite of the spine's cell, not as a cell of its own.""" + from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + ) + + config = UKFullBuildConfig(calibration_year=2026) + graph = uk_full_graph( + config, + atomic_geography_definition=uk_atomic_assignment_definition( + toy_support_payloads(), seed=config.seed + ), + ).graph + assert ("household", GEOGRAPHY_HOUSEHOLD_KEY_COLUMN) in { + (owned.entity, owned.column) + for owned in graph.node("geographic_identity").outputs + } + (identity,) = graph.node("uk.full.identity").outputs + assert (identity.entity, identity.column, identity.rewrite) == ( + "household", + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + True, + ) + + def test_local_surface_selection_keeps_exact_target_periods(): rows = pd.DataFrame( { diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_expand.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_expand.py index 0b338321c9..7554894ae2 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_expand.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_expand.py @@ -2,25 +2,49 @@ G1 clones every FRS household K times through the shared clone expansion core under the UK's declaration, writing each household's source columns first. -These tests hold it to the clone-first stand-in (``clone_first_frame`` in +The spine's ``geographic_expand`` stage runs it, under its packaged +declaration, which every transform here is built from. These tests hold it +to the clone-first stand-in (``clone_first_frame`` in test_support/microcosm_build/uk_clone_first_frames.py, written without the core) at K = 1, 2, 3, 7 and 15, and check the one-clone frame, the clone-major ids, indices, sources and weights at K = 3, the dtypes, the mass record, the -evidence and every refusal. Invented frames only; every refusal prints counts, -names and declared constants, never an id or a weight. +evidence and every refusal. On the spine's graph they check that the stage is +an EXPAND that conserves mass at design weights, that K is a parameter of its +node and never of its stage contract, that one clone is an EXPAND of no row +through the executor, that the node lists each table's new cells in the +order G1 writes them, and that the release's input coverage requires its +mass record. They also check that the source columns are named once, in the +geographic_clones leaf, and that the H2 registry builds G1 and its engine at +its fixture's clone count. Invented frames and the synthetic H2 fixture only; +every refusal prints counts, names and declared constants, never an id or a +weight. """ from __future__ import annotations +import functools +import json import re +import shutil +from dataclasses import replace +from pathlib import Path +from types import MappingProxyType import numpy as np import pandas as pd import pytest from microcosm.build.clone_expansion import CloneDeclaration, CloneIdRule -from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.uk_runtime import copy_id_table, geographic_expand +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.uk_runtime import ( + copy_id_table, + geographic_clones, + geographic_expand, + graph_kernels, + rowwise_dataset, + spi_support, +) from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.geographic_clones import ( geographic_clone_count, @@ -30,6 +54,7 @@ GEOGRAPHIC_CLONE_LIMIT, GEOGRAPHIC_COPY_ID_FIELD, GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON, + GEOGRAPHIC_EXPAND_OPERATION_KIND, GEOGRAPHIC_EXPAND_STAGE_NAME, SOURCE_HOUSEHOLD_KEY_COLUMN, SOURCE_YEAR_COLUMN, @@ -37,9 +62,12 @@ UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, UKGeographicExpandStageTransform, expand_uk_geographic_clones, + geographic_expand_operation_parameters, mint_geographic_clone_ids, require_room_for_geographic_clones, ) +from microcosm.build.uk_runtime.graph import uk_spine_graph +from microcosm.build.uk_runtime.graph_kernels import UKExpandStageKernel from microcosm.build.uk_runtime.household_lineage import ( GEOGRAPHIC_SUPPORT_BRANCH, UK_COPY_LINEAGE, @@ -51,13 +79,20 @@ validate_uk_national_frame, ) from microcosm.build.uk_runtime.receipt_counts import rounded_count +from microcosm.build.uk_runtime.release_input_coverage import ( + _family_build_state_diagnostics, + load_uk_release_input_coverage_manifest, +) from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN +from microcosm.graph import KernelContext, StructuralDelta +from microcosm.graph.population import Population, patch from test_support.microcosm_build.uk_clone_first_frames import ( base_id_tables, clone_first_frame, with_last_household_moved, ) +from test_support.paths import paths_for _CLONE_COUNTS = (1, 2, 3, 7, 15) #: The five id columns, by table. @@ -95,19 +130,11 @@ def _base_frame(households: int = 12, *, time_period: str = _PERIOD) -> Frame: def _stage(**overrides: object) -> SourceStageSpec: - """A stand-in stage for G1: no stage declares it until the spine wires it.""" + """G1's packaged stage declaration, with ``overrides`` replacing fields.""" - fields: dict[str, object] = { - "stage": GEOGRAPHIC_EXPAND_STAGE_NAME, - "survey": "Invented test stage", - "source": "Invented test stage", - "grain": "household", - "artifacts": (), - "operations": (), - "outputs": UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, - } - fields.update(overrides) - return SourceStageSpec(**fields) + spec = load_country_spec("uk") + assert spec.sources is not None + return replace(spec.sources.stage_map()[GEOGRAPHIC_EXPAND_STAGE_NAME], **overrides) def _with_column(frame: Frame, entity: str, column: str) -> Frame: @@ -364,6 +391,312 @@ def test_the_declaration_is_the_copy_id_tables_field_through_named_functions() - ) +# --- the spine's stage -------------------------------------------------------------- + + +@functools.cache +def _spine(clones: int = 1): + return uk_spine_graph(geographic_clones=clones) + + +def _context(node, frame: Frame) -> KernelContext: + """What the executor hands G1's node on ``frame``: its tables, household + weights and strata, and the node's own parameters.""" + + return KernelContext( + node=node, + tables=MappingProxyType( + {entity: frame.table(entity) for entity in frame.entities} + ), + weights=MappingProxyType({"household": frame.weights_for("household")}), + strata=frame.strata, + params=node.params, + rng=np.random.default_rng(0), + sources=MappingProxyType({}), + artifacts={}, + tolerances={}, + numerics={}, + ) + + +def test_the_stage_is_an_expand_that_conserves_mass_at_design_weights() -> None: + graph = _spine() + node = graph.node(GEOGRAPHIC_EXPAND_STAGE_NAME) + assert node.structural is StructuralDelta.EXPAND + assert node.kernel == f"uk.stage.expand.{GEOGRAPHIC_EXPAND_STAGE_NAME}@1" + # Person mass is conserved, to the 1/K split's rounding, and the clones + # keep the root's design weights. + assert node.mass == "conserve" + assert node.params["expand_weight_entity"] == "household" + assert node.params["expand_weight_kind"] == WeightKind.DESIGN.value + cells = { + column: (entity, dtype) for entity, column, dtype in node.params["expand_cells"] + } + assert cells == { + **{ + ladder_clone_index_column(entity): (entity, "int64") + for entity in UK_NATIONAL_SCHEMA.entities + }, + SOURCE_HOUSEHOLD_ID_COLUMN: ("household", "int64"), + SOURCE_YEAR_COLUMN: ("household", "int64"), + SOURCE_HOUSEHOLD_KEY_COLUMN: ("household", "string"), + } + assert set(cells) == set(UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS) + # Its claim node owns exactly the cells it materializes. + owned = graph.node(f"{GEOGRAPHIC_EXPAND_STAGE_NAME}.owned") + assert {(cell.entity, cell.column) for cell in owned.outputs} == { + (entity, column) for column, (entity, _) in cells.items() + } + + +def test_k_is_a_parameter_of_the_stage_s_node_never_of_its_contract() -> None: + """K enters G1's node key, and through the chain every later node's, but + no declaration: the stage's contract and every other node are the same + at every K.""" + + one = _spine(1) + for clones in (1, 3, GEOGRAPHIC_CLONE_LIMIT): + graph = _spine(clones) + node = graph.node(GEOGRAPHIC_EXPAND_STAGE_NAME) + assert node.params["geographic_clones"] == clones + assert {**node.params, "geographic_clones": 1} == dict( + one.node(GEOGRAPHIC_EXPAND_STAGE_NAME).params + ) + assert [ + item.id for item in graph.nodes if "geographic_clones" in item.params + ] == [GEOGRAPHIC_EXPAND_STAGE_NAME] + changed = {item.id for item in graph.nodes if item != one.node(item.id)} + assert changed == ({GEOGRAPHIC_EXPAND_STAGE_NAME} if clones != 1 else set()) + # The declared operation is the reviewed one, which names no clone count. + stage = _stage() + declared = tuple( + (operation.kind, dict(operation.parameters)) for operation in stage.operations + ) + assert declared == geographic_expand_operation_parameters() + ((_, parameters),) = declared + assert not {"clones", "clone_count", "geographic_clones"} & set(parameters) + assert not re.search(r"\d", GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON) + + +@pytest.mark.parametrize("clones", [0, GEOGRAPHIC_CLONE_LIMIT + 1, True, 1.0, "2"]) +def test_the_spine_graph_refuses_a_clone_count_outside_the_field_s_bound( + clones: object, +) -> None: + with pytest.raises( + ValueError, + match="geographic_clones must be a whole number from 1 to 100, the " + "geographic_support copy-id field's bound", + ): + uk_spine_graph(geographic_clones=clones) + + +def test_the_spine_graph_refuses_clones_without_the_stage_that_makes_them() -> None: + spec = load_country_spec("uk") + assert spec.sources is not None + root_only = replace( + spec, sources=replace(spec.sources, stages=spec.sources.stages[:1]) + ) + # One clone needs no stage. + assert GEOGRAPHIC_EXPAND_STAGE_NAME not in { + node.id for node in uk_spine_graph(root_only).nodes + } + with pytest.raises(ValueError, match="needs the 'geographic_expand' stage"): + uk_spine_graph(root_only, geographic_clones=2) + + +def test_the_kernel_refuses_a_transform_making_another_clone_count() -> None: + node = _spine(3).node(GEOGRAPHIC_EXPAND_STAGE_NAME) + kernel = UKExpandStageKernel( + GEOGRAPHIC_EXPAND_STAGE_NAME, + UKGeographicExpandStageTransform(_stage(), clones=2), + ) + with pytest.raises( + ValueError, + match=r"declares 3 geographic clone\(s\) on its node, but its transform " + r"makes 2\.$", + ): + kernel.run(_context(node, _base_frame())) + + +@pytest.mark.parametrize("clones", [1, 3]) +def test_one_clone_is_an_expand_of_no_row_through_the_executor(clones: int) -> None: + """The stage's kernel and the executor's patch at K: one clone adds no + row and keeps every incumbent cell and weight, and three clones triple + every table, each conserving the household mass of the frame.""" + + frame = _base_frame() + node = _spine(clones).node(GEOGRAPHIC_EXPAND_STAGE_NAME) + transform = UKGeographicExpandStageTransform(_stage(), clones=clones) + result = UKExpandStageKernel(GEOGRAPHIC_EXPAND_STAGE_NAME, transform).run( + _context(node, frame) + ) + rows = {entity: len(frame.table(entity)) for entity in frame.entities} + assert {entity: len(lineage) for entity, lineage in result.expand.items()} == { + entity: (clones - 1) * count for entity, count in rows.items() + } + assert result.weights.kind is WeightKind.DESIGN + expanded = patch( + Population.from_frame(frame, "base"), replace(node, base="base"), result + ) + assert {entity: len(expanded.frame.table(entity)) for entity in frame.entities} == { + entity: clones * count for entity, count in rows.items() + } + record = expanded.mass_ledger[-1] + assert (record.operation, record.policy) == ("expand", "conserve") + assert np.isclose(record.after_total, record.before_total, rtol=1e-9, atol=0.0) + assert expanded.frame.mass_log[-1].reason == GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON + weights = expanded.frame.weights_for("household") + assert weights.kind is WeightKind.DESIGN + household = expanded.frame.table("household") + assert sorted(set(household[ladder_clone_index_column("household")])) == list( + range(clones) + ) + if clones == 1: + assert ( + weights.values.tobytes() == frame.weights_for("household").values.tobytes() + ) + for entity in frame.entities: + before = frame.table(entity) + pd.testing.assert_frame_equal( + expanded.frame.table(entity)[list(before.columns)], + before, + check_exact=True, + ) + assert household[SOURCE_HOUSEHOLD_ID_COLUMN].tolist() == ( + household["household_id"].tolist() + ) + + +def test_the_release_coverage_family_requires_the_stage_s_mass_record() -> None: + """The release's input coverage has a geographic_expand family + (microcosm#931): it requires the record G1's mass log gains, under the + reason the stage declares, and the record meets the family's check at + every K, one clone included. The family's weight kind is the final spine + frame's, which later stages set, so the check here runs without it.""" + + manifest = load_uk_release_input_coverage_manifest() + family = manifest.family_coverage[GEOGRAPHIC_EXPAND_STAGE_NAME] + assert family["status"] == "required_at_build" + assert family["stage"] == GEOGRAPHIC_EXPAND_STAGE_NAME + assert family["required_mass_change_reason"] == GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON + ((_, parameters),) = geographic_expand_operation_parameters() + assert parameters["reason"] == GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON + alone = replace( + manifest, + family_coverage={ + GEOGRAPHIC_EXPAND_STAGE_NAME: {**family, "output_weight_kind": ""} + }, + ) + for clones in _CLONE_COUNTS: + frame = expand_uk_geographic_clones( + _with_earlier_mass_record(_base_frame()), clones=clones + ).frame + diagnostics, failures = _family_build_state_diagnostics(frame, alone) + assert failures == [], clones + details = diagnostics[GEOGRAPHIC_EXPAND_STAGE_NAME] + assert details["matching_mass_change_records"] == 1, clones + assert details["valid_mass_change_records"] == 1, clones + + +@pytest.mark.parametrize("clones", [1, 3]) +def test_the_node_lists_each_table_s_new_cells_in_the_order_g1_writes_them( + clones: int, +) -> None: + """The graph appends an EXPAND's new cells in its node's ``expand_cells`` + order, and the H2 parity test compares column order, so the node lists + each table's cells in the order the transform appends them: the + household's ``source_year`` and ``source_household_key``, which G1 + writes before the core runs, then the core's source id and each table's + clone index (microcosm#931).""" + + frame = _base_frame() + expanded = expand_uk_geographic_clones(frame, clones=clones).frame + cells = _spine(clones).node(GEOGRAPHIC_EXPAND_STAGE_NAME).params["expand_cells"] + for entity in frame.entities: + before = list(frame.table(entity).columns) + appended = [ + column for column in expanded.table(entity).columns if column not in before + ] + assert appended == [column for owner, column, _ in cells if owner == entity] + + +def test_the_source_columns_are_named_once_in_the_geographic_clones_leaf() -> None: + """G1's source columns are named in the geographic_clones leaf, which + imports nothing first-party, so the leaf spells the source id itself: it + is the Frame's source column, which the shared core writes. G1, the SPI + support channel and the dense line's crosswalk read the leaf's own tuple + (microcosm#931).""" + + assert geographic_clones.UK_GEOGRAPHIC_SOURCE_COLUMNS == ( + SOURCE_HOUSEHOLD_ID_COLUMN, + SOURCE_YEAR_COLUMN, + SOURCE_HOUSEHOLD_KEY_COLUMN, + ) + assert dict(UK_GEOGRAPHIC_CLONES.source_id_columns) == { + "household": SOURCE_HOUSEHOLD_ID_COLUMN + } + for module in (geographic_expand, spi_support, rowwise_dataset): + assert ( + module.UK_GEOGRAPHIC_SOURCE_COLUMNS + is geographic_clones.UK_GEOGRAPHIC_SOURCE_COLUMNS + ), module.__name__ + for name in ("SOURCE_YEAR_COLUMN", "SOURCE_HOUSEHOLD_KEY_COLUMN"): + assert getattr(geographic_expand, name) is getattr(geographic_clones, name) + + +def _fixture_sources(tmp_path: Path, clones: object) -> Path: + """A copy of the H2 fixture's sources whose config gives ``clones`` as + ``geographic_clones``, or no such key where ``clones`` is None.""" + + committed = paths_for("microcosm-graph").tests / "fixtures/parity/uk_spine/sources" + sources = tmp_path / "sources" + shutil.copytree(committed, sources) + descriptor_path = sources / "fixture.json" + descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) + config = descriptor["config"] + assert config["geographic_clones"] == 1 + if clones is None: + del config["geographic_clones"] + else: + config["geographic_clones"] = clones + descriptor_path.write_text(json.dumps(descriptor), encoding="utf-8") + return sources + + +@pytest.mark.parametrize("clones", [1, 3]) +def test_the_fixture_registry_builds_g1_and_its_engine_at_the_config_s_count( + tmp_path: Path, clones: int +) -> None: + """The H2 registry builds G1 at its fixture's config.geographic_clones, + and its engine with that clone count, so the engine refuses a frame of + any other (microcosm#931).""" + + implementations = graph_kernels._fixture_implementations( + _fixture_sources(tmp_path, clones) + ) + assert implementations[GEOGRAPHIC_EXPAND_STAGE_NAME].clones == clones + engines = { + id(transform.engine): transform.engine + for transform in implementations.values() + if hasattr(transform, "engine") + } + assert engines + for engine in engines.values(): + assert type(engine).__name__ == "PolicyEngineUKEngine" + assert engine._clones == clones + + +@pytest.mark.parametrize("clones", [None, True, "1", 1.0]) +def test_the_fixture_registry_refuses_a_clone_count_that_is_not_an_integer( + tmp_path: Path, clones: object +) -> None: + with pytest.raises( + ValueError, + match=r"config\.geographic_clones must be an integer clone count\.$", + ): + graph_kernels._fixture_implementations(_fixture_sources(tmp_path, clones)) + + # --- refusals --------------------------------------------------------------------- @@ -404,7 +737,7 @@ def test_a_frame_with_a_clone_index_source_or_copy_column_is_refused( entity: str, column: str ) -> None: frame = _with_column(_base_frame(), entity, column) - with pytest.raises(ValueError, match="before any copy step") as error: + with pytest.raises(ValueError, match="before any other copy step") as error: expand_uk_geographic_clones(frame, clones=2) assert repr(column) in str(error.value) @@ -550,3 +883,54 @@ def test_a_stage_that_is_not_g1_s_is_refused( ) -> None: with pytest.raises(ValueError, match=message): UKGeographicExpandStageTransform(_stage(**overrides), clones=2) + + +def _reviewed_operation(**changes: object) -> SourceOperationSpec: + """The reviewed operation, with ``changes`` to its parameters.""" + + ((kind, parameters),) = geographic_expand_operation_parameters() + return SourceOperationSpec(kind, {**parameters, **changes}) + + +@pytest.mark.parametrize( + ("operations", "message"), + [ + ( + (_reviewed_operation(reason="An invented reason."),), + r"declaration drifted from the reviewed mapping on parameter\(s\) " + r"\['reason'\]", + ), + # A clone count is the node's, never the declaration's. + ((_reviewed_operation(clones=3),), r"on parameter\(s\) \['clones'\]"), + ((_reviewed_operation(ordinal_bound=10),), r"\['ordinal_bound'\]"), + ( + (_reviewed_operation(weight_kind_out="importance"),), + r"\['weight_kind_out'\]", + ), + ((), "operation order drifted"), + ((_reviewed_operation(), _reviewed_operation()), "operation order drifted"), + ( + (SourceOperationSpec("clone_records", {}),), + f"expected \\('{GEOGRAPHIC_EXPAND_OPERATION_KIND}',\\)", + ), + ], +) +def test_a_declaration_other_than_the_reviewed_operation_is_refused( + operations: tuple[SourceOperationSpec, ...], message: str +) -> None: + with pytest.raises(ValueError, match=message): + UKGeographicExpandStageTransform(_stage(operations=operations), clones=2) + + +def test_a_stage_declaring_an_artifact_is_refused() -> None: + artifact = { + "role": "frs_release", + "kind": "public_aggregate_reference", + "resource": "frs_release.json", + "format": "json", + "runtime_sha256_required": True, + } + with pytest.raises( + ValueError, match="reads no artifact; the stage must declare none, got 1" + ): + UKGeographicExpandStageTransform(_stage(artifacts=(artifact,)), clones=2) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_identity.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_identity.py index 330060c509..df080e7de8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_identity.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_identity.py @@ -2,10 +2,14 @@ G1b keys every household of the geographic expansion's frame by its source, FRS vintage, true source and copy path, as the dense line's -``uk.full.identity`` keys it. These tests hold clone 0's key, and clone k's, -to the dense identity kernel itself on the same households, and check the -clone path, uniqueness, nesting across K, the dtype, the vintage read from -frs_release.json, the evidence and every refusal. Invented frames only; no +``uk.full.identity`` keys it. The spine's ``geographic_identity`` stage runs +it, under its packaged declaration, which every transform here is built from. +These tests hold clone 0's key, and clone k's, to the dense identity kernel +itself on the same households, and check the clone path, uniqueness, nesting +across K, the dtype, the vintage read from frs_release.json, the evidence and +every refusal, a declaration other than the reviewed one's included. On the +spine's graph they check what the stage's node reads and writes, and that +frs_release.json's bytes are in its stage contract. Invented frames only; no refusal prints a key, an id or a year of a row. """ @@ -19,25 +23,32 @@ import pandas as pd import pytest -from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.country_spec import load_country_spec +from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec from microcosm.build.uk_runtime.atomic_area_support import IDENTITY_COLUMN from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key from microcosm.build.uk_runtime.frs_release import load_uk_frs_release from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.geographic_expand import ( + GEOGRAPHIC_EXPAND_STAGE_NAME, SOURCE_HOUSEHOLD_KEY_COLUMN, SOURCE_YEAR_COLUMN, expand_uk_geographic_clones, ) from microcosm.build.uk_runtime.geographic_identity import ( + FRS_RELEASE_ARTIFACT_ROLE, + FRS_RELEASE_RESOURCE, GEOGRAPHIC_IDENTITY_INPUTS, + GEOGRAPHIC_IDENTITY_OPERATION_KIND, GEOGRAPHIC_IDENTITY_SOURCE, GEOGRAPHIC_IDENTITY_STAGE_NAME, UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS, UKGeographicIdentityStageTransform, frs_vintages_by_source_year, + geographic_identity_operation_parameters, key_uk_households_by_clone_path, ) +from microcosm.build.uk_runtime.graph import uk_spine_graph from microcosm.build.uk_runtime.graph_population import ( UKGeographyIdentityKernel, append_uk_population_nodes, @@ -47,6 +58,7 @@ from microcosm.build.uk_runtime.receipt_counts import rounded_count from microcosm.frame import Frame from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN +from microcosm.graph import StructuralDelta from microcosm.graph.population import dtype_for_token from test_support.microcosm_build.uk_clone_first_frames import base_id_tables @@ -108,19 +120,13 @@ def _with_household(frame: Frame, household: pd.DataFrame) -> Frame: def _stage(**overrides: object) -> SourceStageSpec: - """A stand-in stage for G1b: no stage declares it until the spine wires it.""" + """G1b's packaged stage declaration, with ``overrides`` replacing fields.""" - fields: dict[str, object] = { - "stage": GEOGRAPHIC_IDENTITY_STAGE_NAME, - "survey": "Invented test stage", - "source": "Invented test stage", - "grain": "household", - "artifacts": (), - "operations": (), - "outputs": UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS, - } - fields.update(overrides) - return SourceStageSpec(**fields) + spec = load_country_spec("uk") + assert spec.sources is not None + return replace( + spec.sources.stage_map()[GEOGRAPHIC_IDENTITY_STAGE_NAME], **overrides + ) def _dense_keys(household: pd.DataFrame) -> list[str]: @@ -316,6 +322,47 @@ def test_the_transform_runs_the_step_and_records_its_evidence() -> None: assert transform.output_columns() == (IDENTITY_COLUMN,) +# --- the spine's stage -------------------------------------------------------------- + + +def test_the_stage_s_node_reads_g1_s_lineage_and_writes_the_key() -> None: + graph = uk_spine_graph() + node = graph.node(GEOGRAPHIC_IDENTITY_STAGE_NAME) + assert node.kernel == f"uk.stage.{GEOGRAPHIC_IDENTITY_STAGE_NAME}@1" + assert node.structural is StructuralDelta.NONE + read = {(item.entity, column) for item in node.inputs for column in item.columns} + assert {("household", column) for column in GEOGRAPHIC_IDENTITY_INPUTS} <= read + assert [(cell.entity, cell.column, cell.dtype) for cell in node.outputs] == [ + ("household", IDENTITY_COLUMN, "string") + ] + assert UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS == (IDENTITY_COLUMN,) + # The key follows the clones, before any stage that copies households. + order = [item.id for item in graph.nodes] + assert order.index(GEOGRAPHIC_EXPAND_STAGE_NAME) < order.index( + GEOGRAPHIC_IDENTITY_STAGE_NAME + ) + + +def test_the_frs_release_s_bytes_are_in_the_stage_s_contract() -> None: + """The release pins the vintage each key names, so a release of other + bytes re-keys the stage through its contract, and the clones, which read + no resource, keep theirs.""" + + spec = load_country_spec("uk") + moved = replace( + spec, + resource_hashes={**spec.resource_hashes, FRS_RELEASE_RESOURCE: "0" * 64}, + ) + assert spec.resource_hashes[FRS_RELEASE_RESOURCE] != "0" * 64 + graphs = (uk_spine_graph(spec), uk_spine_graph(moved)) + identity, clones = ( + [graph.node(name).params["stage_contract_sha256"] for graph in graphs] + for name in (GEOGRAPHIC_IDENTITY_STAGE_NAME, GEOGRAPHIC_EXPAND_STAGE_NAME) + ) + assert identity[0] != identity[1] + assert clones[0] == clones[1] + + # --- refusals --------------------------------------------------------------------- @@ -407,3 +454,73 @@ def test_a_stage_that_is_not_g1b_s_is_refused( ) -> None: with pytest.raises(ValueError, match=message): UKGeographicIdentityStageTransform(_stage(**overrides)) + + +def _reviewed_operation(**changes: object) -> SourceOperationSpec: + """The reviewed operation, with ``changes`` to its parameters.""" + + ((kind, parameters),) = geographic_identity_operation_parameters() + return SourceOperationSpec(kind, {**parameters, **changes}) + + +@pytest.mark.parametrize( + ("operations", "message"), + [ + ( + (_reviewed_operation(source="lcfs"),), + r"declaration drifted from the reviewed mapping on parameter\(s\) " + r"\['source'\]", + ), + ( + (_reviewed_operation(vintage_resource="vintages.yaml"),), + r"\['vintage_resource'\]", + ), + ((_reviewed_operation(clones=3),), r"\['clones'\]"), + ((), "operation order drifted"), + ( + (SourceOperationSpec("derive", {}),), + f"expected \\('{GEOGRAPHIC_IDENTITY_OPERATION_KIND}',\\)", + ), + ], +) +def test_a_declaration_other_than_the_reviewed_operation_is_refused( + operations: tuple[SourceOperationSpec, ...], message: str +) -> None: + with pytest.raises(ValueError, match=message): + UKGeographicIdentityStageTransform(_stage(operations=operations)) + + +def _release_artifact(**changes: object) -> dict[str, object]: + (artifact,) = _stage().artifacts + return {**artifact, **changes} + + +@pytest.mark.parametrize( + ("artifacts", "message"), + [ + ((), r"got 0 artifact\(s\), 0 of them that one"), + ( + (_release_artifact(runtime_sha256_required=False),), + r"got 1 artifact\(s\), 0 of them that one", + ), + ( + (_release_artifact(resource="frs_release_2023.json"),), + r"got 1 artifact\(s\), 0 of them that one", + ), + ( + (_release_artifact(role="count_resource"),), + r"got 1 artifact\(s\), 0 of them that one", + ), + ( + (_release_artifact(), _release_artifact(role="count_resource")), + r"got 2 artifact\(s\), 1 of them that one", + ), + ], +) +def test_a_stage_without_the_frs_release_as_its_one_artifact_is_refused( + artifacts: tuple[dict[str, object], ...], message: str +) -> None: + with pytest.raises(ValueError, match=message) as error: + UKGeographicIdentityStageTransform(_stage(artifacts=artifacts)) + assert repr(FRS_RELEASE_ARTIFACT_ROLE) in str(error.value) + assert repr(FRS_RELEASE_RESOURCE) in str(error.value) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index a650e5f412..19b2233cfd 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -85,8 +85,10 @@ def test_uk_spine_graph_contains_every_manifest_stage() -> None: # (PolicyEngine/uk-autumn-budget-dashboard-2026#316); and microcosm#1490 # spi_employment_shell after company_cars, then aps_occupation; and # microcosm#1095 (P1) pip_points just before trading_allowance_receipts, - # and (Y1) uc_limited_capability just before uc_take_up. - assert len(expected) == 56 + # and (Y1) uc_limited_capability just before uc_take_up; and + # microcosm#931 geographic_expand and geographic_identity after + # frs_annual_income_spells, before frs_take_up. + assert len(expected) == 58 assert {"frs_hmrc_retained_leaves", "hmrc_spi_income"}.isdisjoint(expected) assert set(expected) <= ids assert {"frs_hmrc_retained_leaves", "hmrc_spi_income"}.isdisjoint(ids) @@ -95,9 +97,15 @@ def test_uk_spine_graph_contains_every_manifest_stage() -> None: for owned in graph.node("create_uk_frs").outputs } assert root_dtypes[("person", "is_uc_claimant")] == "bool" - assert { + expanding = { node.id for node in graph.nodes if node.structural is StructuralDelta.EXPAND - } == UK_SPINE_STRUCTURAL_STAGES + } + assert expanding == UK_SPINE_STRUCTURAL_STAGES + # The geographic clones expand the frame; their keys are a column the + # clones carry (microcosm#931). + assert "geographic_expand" in expanding + assert "geographic_identity" not in expanding + assert graph.node("geographic_identity").structural is StructuralDelta.NONE def test_uk_spine_compile_order_is_derived_from_declared_inputs() -> None: @@ -461,7 +469,10 @@ def changed_helper_bytes(path: Path) -> bytes: return content monkeypatch.setattr(Path, "read_bytes", changed_helper_bytes) - assert len(affected) == 10 + # Eleven since the geographic clones mint their ids in the table's + # geographic_support field (microcosm#931). + assert "geographic_expand" in stages + assert len(affected) == 11 for ref in affected: assert registry.implementation_hash(ref) != before[ref], ref for ref in controls: @@ -1228,14 +1239,20 @@ def changed_module_bytes(path): #: they find, so a new one is bound or listed here with its reason. _STAGE_UNBOUND_NAMES: dict[tuple[str, str], str] = { ("microcosm.frame.concepts", "SOURCE_HOUSEHOLD_ID_COLUMN"): ( - "names the source household column, CopyLineage's default source, " - "which the support channel writes by its literal name. A node is " - "given only its declared columns, so under a renamed constant the " - "lineage readers would find no source column, which they refuse " - "unless counting before the copies on a frame without a copy column. " - "A spine stage that reads it declares the column by that name, which " - "its key hashes, and counts before the copies only beside a copy " - "column, or declares neither, and counts as before (checked)" + "names the source household column, CopyLineage's default source. " + "geographic_expand writes it through the constant and declares it by " + "its literal name, in its outputs, which its contract hashes, and in " + "its graph cell, and its transform refuses a stage whose declared " + "outputs are not the columns it writes. So under a renamed constant " + "the spine build refuses it before any node runs, and the declaration " + "edit the rename needs re-keys geographic_expand and every node after " + "it. A node is given only its declared columns: every stage after " + "geographic_expand that reads the name is given the column, declared " + "by that name or as an open surface, and one that counts before the " + "copies is given the geographic clone index too; the one stage before " + "it that reads the name, frs_annual_income_spells, is given neither " + "the column nor any copy column, and counts each household as its own " + "source (checked; microcosm#931)" ), } #: The same for the UK graph kernels: names a kernel reads from a module its @@ -1250,18 +1267,21 @@ def changed_module_bytes(path): for name in ("LAD23_NAMES_SHA256", "LAD23_NAMES_URL") }, ("microcosm.build.uk_runtime.geographic_clones", "ladder_clone_index_column"): ( - "names the geographic clone index columns, which only the dense expand " - "writes, and uk.full.expand@1 hashes geographic_clones. The calibrated " - "gates' support-family fold reads the household's on the expanded " - "frame, so their key chains on the expand's; the national gates fold " - "the national frame, which no ladder expands, and the fold keys on the " - "column only where the table carries it " - "(diagnostics.uk_support_family_weights). The dense measures read it " - "only to split their frame into the engine blocks their engine_blocks " - "parameter asks for, and refuse a frame without it; their population " - "is the expand's, so their key chains on it too. The national problem " - "resolves its measures in one block, which never reads it " - "(full_measure._engine_block_frames; checked)" + "names the geographic clone index columns. geographic_expand writes " + "them on every spine from microcosm#931 group (b) on, and the dense " + "expand on a spine built before it, and both keys hash " + "geographic_clones: geographic_expand counts by lineage, and " + "uk.full.expand@1 hashes it beside rowwise_dataset. The national " + "gates, whose support-family fold reads them on the national frame, " + "hash it too. The calibrated gates' support-family fold reads the " + "household's on the dense frame, and the dense line expands only a " + "spine without them until group (c), so their key chains on the " + "expand's. The dense measures read it only to split their frame into " + "the engine blocks their engine_blocks parameter asks for, and refuse " + "a frame without it; their population is the expand's, so their key " + "chains on it too. The national problem resolves its measures in one " + "block, which never reads it (full_measure._engine_block_frames; " + "checked)" ), } #: Where the kernel rule stops, each with why a kernel that names the @@ -1597,30 +1617,184 @@ def upstream(node_id: str) -> set[str]: assert set(unlisted) == set(_STAGE_UNBOUND_NAMES), sorted( set(unlisted) ^ set(_STAGE_UNBOUND_NAMES) ) - # The source household column's reason. The support channel writes it by - # its literal name; a stage that reads it declares it by that name and - # counts before the copies only beside a copy column, or declares neither - # it nor a copy column. - from microcosm.build.uk_runtime import spi_support + # The source household column's reason. geographic_expand declares it by + # its literal name, in its outputs and its graph cell, and its transform + # refuses other outputs (test_uk_geographic_expand). Every stage after it + # that reads the name is given the column, and the one before it neither + # the column nor a copy column, counting before the copies; a stage after + # it that counts before the copies is held to the clone index too + # (test_clone_lineage_reaches_every_stage_counting_before_the_copies). + from microcosm.build.uk_runtime.geographic_expand import ( + GEOGRAPHIC_EXPAND_STAGE_NAME, + UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, + ) from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN - assert f'"{SOURCE_HOUSEHOLD_ID_COLUMN}"' in Path(spi_support.__file__).read_text() + spec = load_country_spec("uk") + assert spec.sources is not None + declared = spec.sources.stage_map() + roster = [stage.stage for stage in spec.sources.stages] + clones = roster.index(GEOGRAPHIC_EXPAND_STAGE_NAME) + assert declared[GEOGRAPHIC_EXPAND_STAGE_NAME].outputs == ( + UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS + ) + assert SOURCE_HOUSEHOLD_ID_COLUMN in UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS + assert ("household", SOURCE_HOUSEHOLD_ID_COLUMN) in { + (owned.entity, owned.column) + for owned in graph.node(f"{GEOGRAPHIC_EXPAND_STAGE_NAME}.owned").outputs + } copy_columns = set(UK_COPY_LINEAGE.copy_columns) readers = unlisted[("microcosm.frame.concepts", "SOURCE_HOUSEHOLD_ID_COLUMN")] - counting = {stage: _counts_before_copies(walks[stage][2]) for stage in readers} - assert any(counting.values()) + before = [stage for stage in readers if roster.index(stage) < clones] + assert before == ["frs_annual_income_spells"] for stage in readers: - read = { - column - for slice_ in graph.node(walks[stage][0]).inputs - if slice_.entity == "household" + read = _household_inputs(graph.node(walks[stage][0])) + if roster.index(stage) > clones: + assert SOURCE_HOUSEHOLD_ID_COLUMN in read, stage + elif stage in before: + assert not read & ({SOURCE_HOUSEHOLD_ID_COLUMN} | copy_columns), stage + assert _counts_before_copies(walks[stage][2]), stage + # The geography key's name is the geographic_clones leaf's, which + # geographic_identity hashes as a counting stage, so the stage reads it + # bound; the dense line's atomic_area_support re-exports it, and the stage + # reaches nothing of that module. + from microcosm.build.uk_runtime import atomic_area_support, geographic_clones + from microcosm.build.uk_runtime.geographic_identity import ( + GEOGRAPHIC_IDENTITY_STAGE_NAME, + UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS, + ) + + key = geographic_clones.GEOGRAPHY_HOUSEHOLD_KEY_COLUMN + assert atomic_area_support.IDENTITY_COLUMN is key + assert UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS == (key,) + assert declared[GEOGRAPHIC_IDENTITY_STAGE_NAME].outputs == (key,) + identity_node, identity_hashed, identity_walk = walks[ + GEOGRAPHIC_IDENTITY_STAGE_NAME + ] + assert geographic_clones.__name__ in identity_hashed + assert ( + geographic_clones.__name__, + "GEOGRAPHY_HOUSEHOLD_KEY_COLUMN", + ) in identity_walk.stops + assert atomic_area_support.__name__ not in identity_walk.reach + assert not any( + module == atomic_area_support.__name__ for module, _ in identity_walk.stops + ) + assert [ + (owned.entity, owned.column) for owned in graph.node(identity_node).outputs + ] == [("household", key)] + + +def _household_inputs(node) -> set[str]: + """The household columns a node is given.""" + + return { + column + for slice_ in node.inputs + if slice_.entity == "household" + for column in slice_.columns + } + + +def test_clone_lineage_reaches_every_stage_counting_before_the_copies() -> None: + """A stage after geographic_expand that counts its receipts' support + before the copies is given each household's true source and its + geographic clone index (microcosm#931). The shared reader counts by the + source column wherever the frame carries it and refuses the clone index + without it, but a node given neither counts every clone as a source of + its own, and the source column's check passes such a stage: this rule + catches it. frs_take_up, frs_person_draws and frs_household_draws are + among the stages it holds, so it is not vacuous.""" + + from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column + from microcosm.build.uk_runtime.geographic_expand import ( + GEOGRAPHIC_EXPAND_STAGE_NAME, + ) + from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN + + graph = uk_spine_graph() + walks = _stage_walks() + roster = [stage.stage for stage in load_country_spec("uk").sources.stages] + after = roster[roster.index(GEOGRAPHIC_EXPAND_STAGE_NAME) + 1 :] + counting = [stage for stage in after if _counts_before_copies(walks[stage][2])] + assert {"frs_take_up", "frs_person_draws", "frs_household_draws"} <= set(counting) + lineage = {SOURCE_HOUSEHOLD_ID_COLUMN, ladder_clone_index_column("household")} + unread = { + stage: sorted(lineage - _household_inputs(graph.node(walks[stage][0]))) + for stage in counting + } + assert not any(unread.values()), unread + + +def test_every_stage_that_folds_geographic_clones_reads_the_clone_index() -> None: + """A stage whose fits fold each survey record's geographic clones onto + its lowest-id clone (``household_lineage.fold_geographic_clones``) is + given the household clone index the fold's key reads: a node given none + folds nothing, so its fits would count each clone as a record of its own + (microcosm#931). property_components declares it too, so that a check of + the stage can read its frame's clone count.""" + + from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column + + graph = uk_spine_graph() + walks = _stage_walks() + lineage = "microcosm.build.uk_runtime.household_lineage" + folds = {"fold_geographic_clones", "collapse_geographic_clones"} + folding = sorted( + stage + for stage, (_, _, walk) in walks.items() + if folds & set(walk.reach.get(lineage, ())) + ) + assert {"spi_employment_shell", "spi_housing_shell", "uc_capital_coherence"} <= ( + set(folding) + ) + clone_index = ladder_clone_index_column("household") + unread = [ + stage + for stage in (*folding, "property_components") + if clone_index not in _household_inputs(graph.node(walks[stage][0])) + ] + assert not unread, unread + + +def test_the_raw_tab_readers_and_the_draws_read_nothing_the_other_writes() -> None: + """frs_hmrc_spine_leaves and frs_annual_income_spells moved ahead of + frs_take_up, frs_person_draws and frs_household_draws, with the + geographic clones between them (microcosm#931). The move changes no value: + neither group reads a column the other writes, they write no column in + common, and none of them sets weights.""" + + graph = uk_spine_graph() + moved = ("frs_hmrc_spine_leaves", "frs_annual_income_spells") + draws = ("frs_take_up", "frs_person_draws", "frs_household_draws") + + def reads(stages: tuple[str, ...]) -> set[tuple[str, str]]: + return { + (slice_.entity, column) + for stage in stages + for slice_ in graph.node(stage).inputs for column in slice_.columns } - if SOURCE_HOUSEHOLD_ID_COLUMN not in read: - assert not read & copy_columns, stage - elif not read & copy_columns: - assert not counting[stage], (stage, sorted(counting[stage])) + + def writes(stages: tuple[str, ...]) -> set[tuple[str, str]]: + return { + (owned.entity, owned.column) + for stage in stages + for owned in graph.node(stage).outputs + } + + assert writes(moved) and writes(draws) + assert not reads(moved) & writes(draws) + assert not reads(draws) & writes(moved) + assert not writes(moved) & writes(draws) + for stage in (*moved, *draws): + node = graph.node(stage) + assert node.structural is StructuralDelta.NONE, stage + assert node.weights is None, stage + order = [node.id for node in graph.nodes] + assert max(map(order.index, moved)) < order.index("geographic_expand") + assert order.index("geographic_identity") < min(map(order.index, draws)) def _counts_before_copies(walk: _Walk) -> set[tuple[str, str]]: @@ -2164,6 +2338,7 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: from microcosm.build.uk_runtime import ( geographic_clones, graph_build, + graph_kernels, graph_national, graph_population, graph_targets, @@ -2184,18 +2359,30 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: # before they read the columns' name. population = inspect.signature(graph_targets.append_uk_target_nodes).parameters assert population["population"].default == "uk.full.expand" - # And the premise of their reason, that only the dense expand writes the - # clone index columns: no spine node writes them. Once the spine's - # geographic expansion does, the reason is rewritten with this check. - written = { - (owned.entity, owned.column) - for node in uk_spine_graph().nodes - for owned in node.outputs - } + # And the premises of their reason for the writers. On the spine, the + # geographic expansion alone writes the clone index columns, and its key + # hashes geographic_clones; the spine build installs it at one clone. The + # dense expand refuses a spine carrying them + # (test_dense_refuses_a_clone_first_spine_at_any_clone_count). + from microcosm.build.uk_runtime import spine_build + from microcosm.build.uk_runtime.graph import uk_registry + + spine = uk_spine_graph() + writers: dict[tuple[str, str], set[str]] = {} + for node in spine.nodes: + for owned in node.outputs: + writers.setdefault((owned.entity, owned.column), set()).add(node.id) for entity in ("person", "benunit", "household"): - assert (entity, geographic_clones.ladder_clone_index_column(entity)) not in ( - written - ) + assert writers[ + (entity, geographic_clones.ladder_clone_index_column(entity)) + ] == {"geographic_expand.owned"} + expand = uk_registry(graph=spine).get(spine.node("geographic_expand").kernel) + assert geographic_clones.__name__ in hashed_module_names(expand, graph_kernels) + assert spine_build._GEOGRAPHIC_CLONES == 1 + # The national gates fold the national frame, which carries the columns + # from a clone-first spine, so their key hashes the leaf. + national_gates = kernels["uk.full.national-gates@1"] + assert geographic_clones.__name__ in _hashed_modules(national_gates) def calls(module, name: str) -> list: tree = ast.parse(Path(module.__file__).read_text(encoding="utf-8")) @@ -3537,3 +3724,61 @@ def test_spi_benefit_coherence_reads_cover_its_work_tests() -> None: ) } | {("household", "source_household_id")} assert required <= declared, sorted(required - declared) + + +def test_the_root_types_its_area_carriers_and_geographic_expand_owns_the_sources() -> ( + None +): + """microcosm#931: the root's IMD decile is an int64 cell and its band flag + a bool one; the households' source columns are geographic_expand's cells, + and spi_support_channel, whose copies inherit them, owns none of them.""" + from microcosm.build.uk_runtime.frs_spine import ( + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, + FRS_IMD_DECILE_COLUMN, + ) + + graph = uk_spine_graph() + root = { + (owned.entity, owned.column): owned.dtype + for owned in graph.node("create_uk_frs").outputs + } + assert root[("household", FRS_IMD_DECILE_COLUMN)] == "int64" + assert root[("household", FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN)] == "bool" + + sources = { + ("household", column) + for column in ("source_household_id", "source_year", "source_household_key") + } + expand = { + (owned.entity, owned.column) + for owned in graph.node("geographic_expand.owned").outputs + } + support = { + (owned.entity, owned.column) + for owned in graph.node("spi_support_channel.owned").outputs + } + assert sources <= expand + assert sources.isdisjoint(support) + + +def test_microdf_version_joins_every_spine_nodes_numerical_dependencies() -> None: + """microcosm#931, O7: microdf-python's installed version ("not-installed" + without it) is a numerical dependency of the root and of every stage, so + a microdf change re-keys the spine; it is bound, not declared, until the + next lock bump.""" + from importlib import metadata + + from microcosm.build.uk_runtime.graph import _numerical_dependency_versions + + try: + installed = metadata.version("microdf-python") + except metadata.PackageNotFoundError: + installed = "not-installed" + versions = dict(_numerical_dependency_versions()) + assert versions["microdf-python"] == installed + + graph = uk_spine_graph() + bound = [node for node in graph.nodes if "numerical_dependencies" in node.params] + assert {node.id for node in bound} >= {"create_uk_frs", "geographic_expand"} + for node in bound: + assert dict(node.params["numerical_dependencies"]) == versions, node.id diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py index c97e9e133f..d621a5bcb2 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py @@ -362,10 +362,19 @@ def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions( } # The gate admits whichever stage follows the BRMA checkpoint in the roster # (the SPI block since #1012 moved it ahead of was_wealth), as main's tool - # does with UK_SPINE_ASSEMBLED_FINAL_STAGE. + # does with UK_SPINE_ASSEMBLED_FINAL_STAGE. Since the HMRC leaves and the + # annual income spells moved ahead of the geographic clones, which the + # draws and the BRMA follow, that is the SPI support channel + # (microcosm#931). stages = endpoint.stage_names admitted = stages[stages.index("frs_brma") + 1] - assert admitted == "frs_hmrc_spine_leaves" + assert admitted == "spi_support_channel" + assert { + "frs_hmrc_spine_leaves", + "frs_annual_income_spells", + "geographic_expand", + "geographic_identity", + } <= set(stages[: stages.index("frs_brma")]) assert "spine.gates.assembled" in compiled.predecessors["frs_brma.checkpoint"] assert "spine.gates.assembled" in compiled.predecessors[admitted] assert graph.node(admitted).params["spine_gate_phase"] == "assembled" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py index 90e9d25ac7..6edef754a3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_high_value_home_support.py @@ -574,9 +574,10 @@ def test_the_copy_rule_names_the_one_exception() -> None: ((_, payload),) = hv.high_value_support_operation_parameters() expected = ( "every selected household becomes copies rows at weight / copies, " - "except where none of the 64 heaviest households left whole can carry " - "the exact-total correction, when the shared fallback can put it on a " - "root or copy, off weight / copies by about one ulp of the total weight" + "except where none of the 64 heaviest households left whole, tried " + "heaviest first, tied weights in row order, can carry the exact-total " + "correction, when the shared fallback can put it on a root or copy, off " + "weight / copies by about one ulp of the total weight" ) assert CGT_SUPPORT_CORRECTION_CANDIDATES == 64 assert payload["copy_rule"] == expected diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index b0c417f20e..8fccd129d3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -3145,6 +3145,15 @@ def test_a_copy_off_its_lineage_fails_closed_by_count(self) -> None: "declares only the incidence clone's flag, which it pairs on, and " "creates no rows" ), + "geographic_expand": ( + "declares no flag: its clones carry a clone index, and every receipted " + "stage ran on all of them. They copy only the values of the stages " + "before it, frs_spine to frs_education_grant_split, " + "frs_hmrc_spine_leaves and frs_annual_income_spells, and no E4 to E9 " + "receipt recomputes a draw of those; E4 recomputes frs_take_up, " + "frs_person_draws, frs_household_draws and frs_brma, which run on " + "every clone (microcosm#931)" + ), } @@ -3185,3 +3194,14 @@ def test_the_stacking_map_names_every_structural_stage_flag() -> None: operation.get("clone_flag_column") for operation in stages["cgt_incidence_anchor"]["operations"] } - {None} == {"household_is_capital_gains_clone"} + assert declared["geographic_expand"] == [] + names = [stage["stage"] for stage in tool._source_stage_roster()] + assert names.index("geographic_expand") < min( + names.index(stage) + for stage in ( + "frs_take_up", + "frs_person_draws", + "frs_household_draws", + "frs_brma", + ) + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py index 5f8ae0570c..6dabc198a8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py @@ -238,9 +238,12 @@ def test_the_sidecar_records_batches_only_for_a_final_frame_that_batches( assert spine_build._rules_engine_record(engine, frame) == provenance assert _sidecar(engine, frame)["rules_engine"] == provenance batched = _BatchingEngine(dict(_BATCHES)) + # Beside the batches, microdf's version (microcosm#931); _install reads + # every package at one version. assert spine_build._rules_engine_record(batched, frame) == { **provenance, "engine_batches": _BATCHES, + "microdf_python": VERSION, } assert _sidecar(batched, frame)["rules_engine"]["engine_batches"] == _BATCHES # The record is asked of the final frame the sidecar describes. @@ -293,9 +296,50 @@ def test_a_resumed_spine_records_the_batches_a_fresh_run_does( "pass": SPINE_BATCHED_PASS.name, "declaration_sha256": UK_ENGINE_POPULATION.sha256, }, + "microdf_python": VERSION, } assert _sidecar(fresh, frame)["rules_engine"] == expected assert _sidecar(ran, frame)["rules_engine"] == expected - # One clone, or no clone index (a national spine), records no batches. + # One clone, or no clone index (a national spine), records no batches, + # and no microdf version. for final in (_final_frame((0,)), _final_frame()): - assert "engine_batches" not in _sidecar(fresh, final)["rules_engine"] + record = _sidecar(fresh, final)["rules_engine"] + assert "engine_batches" not in record + assert "microdf_python" not in record + + +def test_microdf_is_recorded_beside_the_batches_alone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Where the engine computes the frame clone by clone, private-school + # attendance stores microdf's rank, so the sidecar records microdf's + # version beside the batches, and "not-installed" when it is absent, as + # the graph binds it (microcosm#931). The engine identity is unchanged. + versions = {"policyengine-uk": VERSION, "microdf-python": "1.3.0"} + + def version(package: str) -> str: + if package not in versions: + raise spine_build.metadata.PackageNotFoundError(package) + return versions[package] + + monkeypatch.setattr(spine_build.metadata, "version", version) + monkeypatch.setattr( + spine_build.metadata, "distribution", lambda package: _Distribution(None) + ) + frame = _final_frame() + batched = _BatchingEngine(dict(_BATCHES)) + provenance = spine_build._rules_engine_provenance() + + assert spine_build._rules_engine_record(batched, frame) == { + **provenance, + "engine_batches": _BATCHES, + "microdf_python": "1.3.0", + } + assert spine_build._rules_engine_record(_BatchingEngine(None), frame) == provenance + del versions["microdf-python"] + assert spine_build._rules_engine_record(batched, frame)["microdf_python"] == ( + "not-installed" + ) + assert _identity(spine_build._rules_engine_provenance()) == _identity( + {"package": "policyengine-uk", "version": VERSION} + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py index eb3d262e35..59cd7a0298 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py @@ -61,9 +61,21 @@ "etb_vat", "etb_services", ] -E7_STAGE_NAMES = [ +#: E7's two stages that read the raw FRS tabs: the HMRC leaves and the annual +#: income spells join them on raw person ids and fill the misses, so they run +#: on the survey's own rows, before the geographic clones (microcosm#931). +E7_SOURCE_ROW_STAGE_NAMES = [ "frs_hmrc_spine_leaves", "frs_annual_income_spells", +] +#: microcosm#931: the geographic clones and their keys, after every stage that +#: reads the raw tabs and before the E4 draws, which run on every clone. +GEOGRAPHIC_CLONE_STAGE_NAMES = [ + "geographic_expand", + "geographic_identity", +] +#: E7's SPI block, which copies the clones' households. +E7_STAGE_NAMES = [ "spi_support_channel", "spi_income_band_donors", "hmrc_spi_income_spine", @@ -150,6 +162,8 @@ "frs_spine", *POST_FRS_SPINE_STAGE_NAMES, *E3_STAGE_NAMES, + *E7_SOURCE_ROW_STAGE_NAMES, + *GEOGRAPHIC_CLONE_STAGE_NAMES, *E4_STAGE_NAMES, # The SPI block runs before the income-conditioned donor imputations, so # E5 and E6 impute onto the SPI rows from their SPI incomes. @@ -218,6 +232,19 @@ def test_e7_block_sits_between_e4_and_e5(self) -> None: *HOUSING_SHELL_STAGE_NAMES, ] + def test_the_geographic_clones_follow_the_raw_tab_readers_and_precede_e4( + self, + ) -> None: + # The HMRC leaves and the annual income spells join the raw FRS tabs on + # raw person ids, so they run on the survey's own rows, and the E4 + # draws then run on every geographic clone (microcosm#931). + canonical = _load_json(CANONICAL_SOURCE_STAGES) + names = [stage["stage"] for stage in canonical["stages"]] + + assert names[ + names.index(E3_STAGE_NAMES[-1]) + 1 : names.index(E4_STAGE_NAMES[0]) + ] == [*E7_SOURCE_ROW_STAGE_NAMES, *GEOGRAPHIC_CLONE_STAGE_NAMES] + def test_e5_and_e6_follow_e7_and_precede_the_uc_rewrites(self) -> None: # uc_reporter_redraw stays after the donor stages: its engine screen # reads the WAS capital proxy where the FRS capital answer is missing. @@ -321,6 +348,8 @@ def test_country_stage_plan_assembles_spine_plan(self) -> None: "etb_services": _identity, "frs_hmrc_spine_leaves": _identity, "frs_annual_income_spells": _identity, + "geographic_expand": _identity, + "geographic_identity": _identity, "spi_support_channel": _identity, "spi_income_band_donors": _identity, "hmrc_spi_income_spine": _identity, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index 94447cb7a3..63509dfda8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -23,6 +23,7 @@ HOUSEHOLD_IS_CGT_SUPPORT_COPY, ) from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity +from microcosm.build.uk_runtime.geographic_expand import expand_uk_geographic_clones from microcosm.build.uk_runtime.high_value_home_support import ( HIGH_VALUE_SUPPORT_COPIES_COLUMN, split_high_value_support_households, @@ -86,7 +87,6 @@ from test_support.microcosm_build.uk_clone_first_frames import ( IdTables, assert_copy_ids_follow_lineage, - clone_first_frame, copy_households, ) from test_support.paths import paths_for @@ -266,10 +266,12 @@ def _clone_first_support_frame( ): """``_support_frame``'s households as geographic clones before stage 16. - Clone ``k`` repeats every row at ``id + k x 10**9`` and an equal share of - its household's weight (``clone_first_frame``), and the SPI support - channel then copies each clone's households, as the clone-first spine will - (microcosm#931). ``tables`` replaces the raw tables ``options`` build. + The spine's geographic expansion (``geographic_expand``) repeats every + row as clone ``k`` at ``id + k x 10**9`` and an equal share of its + household's weight, with the household's source columns, which the SPI + support channel reads, and the channel then copies each clone's + households, as the clone-first spine does (microcosm#931). ``tables`` + replaces the raw tables ``options`` build. """ person, benunit, household = ( @@ -282,7 +284,7 @@ def _clone_first_support_frame( time_period="2024", weight_kind=WeightKind.DESIGN, ) - cloned = engine_tables(clone_first_frame(raw, n_clones)) + cloned = engine_tables(expand_uk_geographic_clones(raw, clones=n_clones).frame) return _spi_support_frame( cloned["person"], cloned["benunit"], diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py index 3e3ec49e9d..a9ea65056b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py @@ -10,6 +10,7 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime import spi_income, spi_spine from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity +from microcosm.build.uk_runtime.geographic_expand import expand_uk_geographic_clones from microcosm.build.uk_runtime.hmrc_income import ( HMRC_SPI_BUILD_PERIOD, HMRC_SPI_INCOME_BAND_LOWER_BOUNDS, @@ -47,7 +48,6 @@ ) from microcosm.build.uk_runtime.terminal_gates import UKZeroWeightStratumDeclaration from microcosm.frame import WeightKind -from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame def _base_frame(*, time_period: str = "2023") -> object: @@ -1229,8 +1229,14 @@ def test_an_expected_count_at_the_source_count_copies_every_household() -> None: def test_clone_zero_of_a_cloned_spine_is_the_one_clone_share(clone_count: int) -> None: from microcosm.build.stochastic_assignment import binomial_share_tolerance - one, one_evidence = _share_stage(clone_first_frame(_share_frame(), 1)) - many, many_evidence = _share_stage(clone_first_frame(_share_frame(), clone_count)) + # The clones as geographic_expand writes them, source columns included, + # which the stage reads and never defaults on a cloned frame (microcosm#931). + one, one_evidence = _share_stage( + expand_uk_geographic_clones(_share_frame(), clones=1).frame + ) + many, many_evidence = _share_stage( + expand_uk_geographic_clones(_share_frame(), clones=clone_count).frame + ) one_household = one.table("household") many_household = many.table("household") @@ -1278,3 +1284,47 @@ def copied(household, clone: int) -> set[int]: rtol=1e-12, atol=0.0, ) + + +def test_the_support_channel_writes_no_source_column_of_its_own() -> None: + """The households' source columns are geographic_expand's (microcosm#931). + + spi_support_channel neither declares them nor writes them: it reads the + ones geographic_expand wrote, keeps them on every base household, and its + copies inherit their roots'. Its declared outputs, its runtime constant + and its graph cells all leave them out. + """ + + from microcosm.build.uk_runtime.geographic_expand import ( + UK_GEOGRAPHIC_SOURCE_COLUMNS, + ) + from microcosm.build.uk_runtime.graph import _STAGE_CELLS + + source = list(UK_GEOGRAPHIC_SOURCE_COLUMNS) + assert source == ["source_household_id", "source_year", "source_household_key"] + stage = _committed_stage("spi_support_channel") + for declared in ( + stage.outputs, + stage.nonnegative_outputs, + UKSPISupportChannelStageTransform.output_columns(), + tuple(cell.column for cell in _STAGE_CELLS["spi_support_channel"]), + ): + assert set(source).isdisjoint(declared) + assert set(source) <= set(_committed_stage("geographic_expand").outputs) + + # 10,000 x 0.01 expected copies over 40 sources: every household is copied. + cloned = expand_uk_geographic_clones(_share_frame(40), clones=2).frame + before = cloned.table("household") + out, _ = _share_stage(cloned) + household = out.table("household") + spi = household["household_support_channel"].eq(SPI_SYNTHETIC_SUPPORT_CHANNEL) + assert spi.any() + pd.testing.assert_frame_equal( + household.loc[~spi, source].reset_index(drop=True), + before[source].reset_index(drop=True), + ) + roots = before.set_index("household_id")[source] + pd.testing.assert_frame_equal( + household.loc[spi, source].reset_index(drop=True), + roots.loc[household.loc[spi, "household_source_id"]].reset_index(drop=True), + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py index a87846dc1c..05c7773bda 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py @@ -1018,11 +1018,15 @@ def _share_frames(n_households: int, *, seed: int = 0): def _cloned_share_tables(n_households: int, *, n_clones: int): - """The share frames as ``n_clones`` geographic clones, clone-major.""" + """The share frames as ``n_clones`` geographic clones, clone-major, as + geographic_expand writes them: each clone's ids, clone indices and source + columns (microcosm#931).""" + from microcosm.build.uk_runtime.geographic_expand import ( + expand_uk_geographic_clones, + ) from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.frame import engine_tables - from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame person, benunit, household = _share_frames(n_households) frame = uk_national_frame( @@ -1032,7 +1036,7 @@ def _cloned_share_tables(n_households: int, *, n_clones: int): time_period="2024", weight_kind=WeightKind.DESIGN, ) - tables = engine_tables(clone_first_frame(frame, n_clones)) + tables = engine_tables(expand_uk_geographic_clones(frame, clones=n_clones).frame) return tables["person"], tables["benunit"], tables["household"] @@ -1356,11 +1360,10 @@ def test_one_indexed_clone_allocates_bit_for_bit_as_no_index() -> None: "state_pension_age": 66, } plain = build_uk_spi_support_channel(person, benunit, household, **arguments) + # One clone as geographic_expand writes it: the frame's own ids and + # weights, clone index 0 and the households' source columns. indexed = build_uk_spi_support_channel( - person, - benunit, - household.assign(household_clone_index=np.zeros(len(household), "int64")), - **arguments, + *_cloned_share_tables(400, n_clones=1), **arguments ) assert indexed.spi_household_ids == plain.spi_household_ids @@ -1497,3 +1500,92 @@ def test_the_derived_clone_stratum_is_not_a_declared_stratum() -> None: expected_spi_households=8, strata_columns=("region", column), ) + + +_SOURCE_COLUMNS = ["source_household_id", "source_year", "source_household_key"] + + +def test_the_channel_keeps_the_source_columns_geographic_expand_wrote() -> None: + # geographic_expand writes every household's source columns before the + # channel runs, so the channel keeps them and each copy carries its root's: + # every clone's copies count the survey household (microcosm#931). + person, benunit, household = _cloned_share_tables(12, n_clones=3) + result = create_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household, + expected_spi_households=None, + source_year=2024, + ) + out = result.household + base = out[support_channel_column("household")].eq(BASE_FRS_SUPPORT_CHANNEL) + + pd.testing.assert_frame_equal( + out.loc[base, _SOURCE_COLUMNS].reset_index(drop=True), + household[_SOURCE_COLUMNS].reset_index(drop=True), + ) + copies = out.loc[~base] + roots = household.set_index("household_id")[_SOURCE_COLUMNS] + pd.testing.assert_frame_equal( + copies[_SOURCE_COLUMNS].reset_index(drop=True), + roots.loc[copies[support_source_id_column("household")]].reset_index(drop=True), + ) + # Every clone's copy of a household names the household's clone-0 id. + assert copies["source_household_id"].nunique() == 12 + assert set(copies["source_household_id"]) == set( + household.loc[household["household_clone_index"].eq(0), "household_id"] + ) + + +@pytest.mark.parametrize( + "dropped", + [ + ("source_household_id",), + ("source_year",), + ("source_household_key",), + ("source_year", "source_household_key"), + tuple(_SOURCE_COLUMNS), + ], +) +def test_a_clone_indexed_table_without_its_source_columns_is_refused( + dropped: tuple[str, ...], +) -> None: + # A clone's own ids are not its source, so on a household table carrying + # the geographic clone index the channel defaults no source column; it + # names the missing columns and no count, id or value (microcosm#931). + person, benunit, household = _cloned_share_tables(4, n_clones=2) + with pytest.raises(ValueError) as caught: + create_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household.drop(columns=list(dropped)), + expected_spi_households=None, + source_year=2024, + ) + missing = [column for column in _SOURCE_COLUMNS if column in dropped] + assert str(caught.value) == ( + "The SPI support channel's households carry 'household_clone_index' but " + f"not {missing}: on the clone-first spine geographic_expand writes every " + "household's source columns, and the stage never takes a clone's own ids " + "for its source (microcosm#931)." + ) + assert not any( + character.isdigit() + for character in str(caught.value).replace("microcosm#931", "") + ) + + # Without the clone index the channel still defaults them: each household + # is its own source. + plain = create_uk_spi_support_tables( + person=person, + benunit=benunit, + household=household.drop(columns=[*dropped, "household_clone_index"]), + expected_spi_households=None, + source_year=2024, + ).household + assert set(_SOURCE_COLUMNS) <= set(plain.columns) + if "source_household_id" in dropped: + base = plain[support_channel_column("household")].eq(BASE_FRS_SUPPORT_CHANNEL) + assert plain.loc[base, "source_household_id"].tolist() == ( + plain.loc[base, "household_id"].tolist() + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_acceptance_receipt.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_acceptance_receipt.py index 70931fb82d..30cdfe4cb9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_acceptance_receipt.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_acceptance_receipt.py @@ -2,7 +2,8 @@ microcosm#771: the previous acceptance evidence quietly described a 24-stage build after the plan had grown to 25. This binder makes that class of drift a -CI failure. The #828/#832/#685 insertions and the #785 and SPI-first reorders are +CI failure. The #828/#832/#685 insertions and the #785, SPI-first and #931 +clone-first reorders are deliberately pending their licensed re-mints, so the historical receipt stays truthful while the test composes only those reviewed transformations. Each transformation pins the receipt state it expects and must be deleted when that re-mint lands. @@ -200,6 +201,22 @@ def _apply_pending_roster_transformations( # last. assert "trading_allowance_receipts" not in roster roster.append("trading_allowance_receipts") + # microcosm#931 re-mint pending: the HMRC leaves and the annual income + # spells, which read the raw FRS tabs by raw person id, move ahead of the + # draws, and the geographic clones and their keys follow them, so the + # draws run on every clone. + assert "geographic_expand" not in roster + assert "geographic_identity" not in roster + raw_tab_readers = ["frs_hmrc_spine_leaves", "frs_annual_income_spells"] + start = roster.index("frs_hmrc_spine_leaves") + assert roster[start : start + len(raw_tab_readers)] == raw_tab_readers + del roster[start : start + len(raw_tab_readers)] + take_up = roster.index("frs_take_up") + roster[take_up:take_up] = [ + *raw_tab_readers, + "geographic_expand", + "geographic_identity", + ] return tuple(roster) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py index 232aee729b..1aed3fe89d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py @@ -496,13 +496,14 @@ def test_export_candidate_columns_strip_ids_and_carry_the_weight() -> None: # in-household care hours carrier (microcosm#1418), the admin UC limited # capability carrier (microcosm#1095, Y1), the company car carriers # (PolicyEngine/uk-autumn-budget-dashboard-2026#316), the value the - # regional home values replaced (microcosm#1112) and the household ISA - # totals policyengine-uk sums from the person balances (microcosm#1415), - # and nothing else. + # regional home values replaced (microcosm#1112), the household ISA totals + # policyengine-uk sums from the person balances (microcosm#1415) and the + # root's area-conditioning carriers (microcosm#931), and nothing else. from microcosm.build.uk_runtime.frs_disability import ( UK_INTERNAL_DISABILITY_REPORTED_COLUMNS, ) from microcosm.build.uk_runtime.frs_spine import ( + FRS_AREA_CONDITIONING_COLUMNS, FRS_BONUS_CARRIER_COLUMN, FRS_CARE_RECIPIENTS_COLUMN, FRS_COMPANY_CAR_CARRIER_COLUMNS, @@ -538,11 +539,69 @@ def test_export_candidate_columns_strip_ids_and_carry_the_weight() -> None: | {f"person.{column}" for column in FRS_COMPANY_CAR_CARRIER_COLUMNS} | {f"household.{RANK_BASIS_COLUMN}"} | {"household.cash_isa", "household.stocks_and_shares_isa"} + | {f"household.{column}" for column in FRS_AREA_CONDITIONING_COLUMNS} ) # A frame without the column passes through untouched. assert uk_release_export_frame(exported) is exported +def test_the_release_boundary_drops_the_root_area_conditioning_carriers() -> None: + """The root's IMD decile and band flag leave at the release boundary + (microcosm#931): they are no engine inputs, and the clone-first spine's + area assignment is their only reader. Every other column stays.""" + import numpy as np + import pandas as pd + + from microcosm.build.uk_runtime.frs_spine import ( + FRS_AREA_CONDITIONING_COLUMNS, + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, + FRS_IMD_DECILE_COLUMN, + ) + from microcosm.build.uk_runtime.national_frame import ( + UK_RELEASE_EXPORT_DROPPED_COLUMNS, + uk_national_frame, + uk_release_export_frame, + ) + from microcosm.build.uk_runtime.terminal_gates import uk_export_candidate_columns + + assert set(FRS_AREA_CONDITIONING_COLUMNS) <= set( + UK_RELEASE_EXPORT_DROPPED_COLUMNS["household"] + ) + frame = uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 2], + "age": [30, 40], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [1, 2], + "region": ["LONDON", "WALES"], + "council_tax_band": ["D", "I"], + FRS_IMD_DECILE_COLUMN: np.asarray([0, 4], dtype="int64"), + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN: [False, True], + } + ), + time_period="2024", + household_weights=np.asarray([1.0, 2.0]), + ) + + exported = uk_release_export_frame(frame) + + household = exported.table("household") + assert set(FRS_AREA_CONDITIONING_COLUMNS).isdisjoint(household.columns) + assert household["council_tax_band"].tolist() == ["D", "I"] + assert { + f"household.{column}" for column in FRS_AREA_CONDITIONING_COLUMNS + }.isdisjoint(uk_export_candidate_columns(exported)) + # The spine keeps them: the boundary drops them from the export alone. + assert set(FRS_AREA_CONDITIONING_COLUMNS) <= set(frame.table("household").columns) + + def test_no_spine_column_writes_an_input_signed_at_the_engine_default() -> None: """A signed engine default (microcosm#1095) holds only while no stage writes it.""" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py index 74f12d6ab3..4210b68d3b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py @@ -149,6 +149,9 @@ def test_graph_scoped_capital_redraw_receives_claimant_roles() -> None: person = frame.table("person") benunit = frame.table("benunit") frame.table("household")["region"] = "LONDON" + # The node reads the household's clone index, since the stage folds + # geographic clones (microcosm#931); this frame is one clone. + frame.table("household")[ladder_clone_index_column("household")] = 0 person["age"] = np.where(person["is_benunit_head"] | person["is_parent"], 40, 5) person[support_channel_column("person")] = person["person_benunit_id"].map( benunit.set_index("benunit_id")[support_channel_column("benunit")] diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/PRODUCED_BY.txt b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/PRODUCED_BY.txt index 3a24f4b36b..7e7fa35336 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/PRODUCED_BY.txt +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/PRODUCED_BY.txt @@ -1 +1 @@ -tools/graph_uk_spine_fixture.py; current 31-transform legacy StagePlan oracle with parsed private-source seams. The acceptance test runs both sides from frs_raw in-process (root weights differ by one ulp between machines); the captured root tables serve the unbound registry's data-only CREATE path. +tools/graph_uk_spine_fixture.py; current legacy StagePlan oracle with parsed private-source seams. The acceptance test runs both sides from frs_raw in-process (root weights differ by one ulp between machines); the captured root tables serve the unbound registry's data-only CREATE path. diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index dc147ab1a0..85d6d2e3a0 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -14,12 +14,13 @@ "source": "policyengine-uk parameters unknown" }, "config": { + "geographic_clones": 1, "qrf_estimators": 4, "spi_donor_sample_size": 64, "spi_sample_fraction": 0.0135, "student_loans_calibration_year": 2025 }, - "description": "Data-only inputs for reconstructing the same 31 current UK stage transform classes used by the legacy StagePlan oracle.", + "description": "Data-only inputs for reconstructing the current UK stage transforms used by the legacy StagePlan oracle.", "inputs": { "aps": "aps.csv", "cgt_asset_type_facts": "cgt_asset_type_facts.json", @@ -563,7 +564,7 @@ ], "grain": "household", "nonnegative_outputs": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then find a family the stage never made, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then count that household as a family the stage never made, which the bracketed family test does not report, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).", "operations": [ { "adult_minimum_age": 16, @@ -573,7 +574,7 @@ "conservation": "exact_total", "copies_column": "cgt_support_copies", "copy_index_column": "cgt_support_copy_index", - "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make", "declared_factor": 1, "expected_support_mass": 145000, "flag_column": "household_is_cgt_support_copy", @@ -711,7 +712,7 @@ "artifacts": [], "grain": "benunit", "nonnegative_outputs": [], - "notes": "Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units before any copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id.", + "notes": "Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units, each geographic clone's on its own ids (microcosm#931), before any other copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units, every geographic clone's included, keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id.", "operations": [ { "assignment_unit": "benunit_id", @@ -1569,8 +1570,8 @@ "locator": "househol.tab", "role": "frs_table", "runtime_sha256_required": true, - "sha256": "7b5fc26572d05bc21cee203b91e20fa9e743c170cabb497a284497b309eb3a89", - "size_bytes": 17554, + "sha256": "b0c0ddc069acdc5f561e725e74eb24250c92275a95c862d2540210c01968814c", + "size_bytes": 18253, "table": "househol", "vintage": "synthetic-h2" }, @@ -1693,8 +1694,8 @@ "locator": "househol.tab", "role": "frs_table", "runtime_sha256_required": true, - "sha256": "7b5fc26572d05bc21cee203b91e20fa9e743c170cabb497a284497b309eb3a89", - "size_bytes": 17554, + "sha256": "b0c0ddc069acdc5f561e725e74eb24250c92275a95c862d2540210c01968814c", + "size_bytes": 18253, "table": "househol", "vintage": "synthetic-h2" }, @@ -2339,8 +2340,8 @@ "locator": "househol.tab", "role": "frs_table", "runtime_sha256_required": true, - "sha256": "7b5fc26572d05bc21cee203b91e20fa9e743c170cabb497a284497b309eb3a89", - "size_bytes": 17554, + "sha256": "b0c0ddc069acdc5f561e725e74eb24250c92275a95c862d2540210c01968814c", + "size_bytes": 18253, "table": "househol", "vintage": "synthetic-h2" } @@ -2636,8 +2637,8 @@ "locator": "househol.tab", "role": "frs_table", "runtime_sha256_required": true, - "sha256": "7b5fc26572d05bc21cee203b91e20fa9e743c170cabb497a284497b309eb3a89", - "size_bytes": 17554, + "sha256": "b0c0ddc069acdc5f561e725e74eb24250c92275a95c862d2540210c01968814c", + "size_bytes": 18253, "table": "househol", "vintage": "synthetic-h2" }, @@ -2751,7 +2752,7 @@ "frs_company_car_fuel_code", "frs_company_car_contribution" ], - "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095).", + "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). imd_decile, each household's decile on its own nation's index of multiple deprivation (IMDE, IMDW, IMDS or IMDN), 1 to 10, and 0 where none is observed or the region is unknown, and council_tax_band_observed, False in Northern Ireland, for a CTBAND code outside 1 to 9, for a band the household's region does not use and for an unknown region (council_tax_band itself is unchanged), are stage-internal carriers the clone-first spine's area assignment conditions on and the release drops; they stay out of every receipt and of the spine sidecar's nonzero shares, and a household tab without one of the four IMD variables is refused (microcosm#931).", "operations": [ { "delimiter": "\t", @@ -2904,11 +2905,13 @@ "liable_for_share_of_household_rent", "household_id", "region", + "imd_decile", "tenure_type", "accommodation_type", "num_bedrooms", "council_tax_reported", "council_tax_band", + "council_tax_band_observed", "council_tax_rebate", "council_tax_single_adult_raw", "water_and_sewerage_charges", @@ -3078,6 +3081,82 @@ "stage": "frs_take_up", "survey": "Family Resources Survey 2024-25" }, + "geographic_expand": { + "artifacts": [], + "grain": "household", + "nonnegative_outputs": [ + "person_clone_index", + "benunit_clone_index", + "household_clone_index", + "source_household_id", + "source_year" + ], + "notes": "Gives every FRS household K geographic clones before any other copy step (microcosm#931), so that each clone can later take an area of its own while together they stand for the survey household once. Clone 0 is the frame's own rows and ids; clone k repeats every person, benefit unit and household with each id minted in the copy-id table's geographic_support field at ordinal k (id_remapping), and every table carries its rows' clone, 0 to K - 1, in its clone index column. Each clone carries an equal share of its household's design weight, the same float in every clone, and the frame's mass log gains one record whose reason names no clone count. Every household carries its source_household_id (its clone-0 id), source_year (the frame's year) and source_household_key, so the stages after it count true sources in survey households. K is a parameter of the stage's graph node, never of this declaration, so the declaration and its contract are the same at every K. The stage runs after frs_hmrc_spine_leaves and frs_annual_income_spells, which join the raw FRS tabs on raw person ids, and before frs_take_up, frs_person_draws, frs_household_draws and frs_brma, which run on every clone. Its evidence gives K, the copy-id field and its radix, the distinct source households rounded by the shared disclosure rules, and the weight split and kind; no row count, id or weight.", + "operations": [ + { + "clone_index_columns": { + "benunit": "benunit_clone_index", + "household": "household_clone_index", + "person": "person_clone_index" + }, + "conservation": "household_total_to_rounding", + "declared_factor": 1, + "entity": "household", + "id_remapping": "copy-id table field geographic_support; clone k takes every entity id + k x the field's radix (10**9)", + "kind": "expand_geographic_clones", + "order": "clone_major", + "ordinal_bound": 100, + "reason": "Geographic clones split each household's design weight equally among its clones, so household mass is conserved.", + "source_columns": [ + "source_household_id", + "source_year", + "source_household_key" + ], + "weight_kind_out": "design", + "weight_split": "equal" + } + ], + "outputs": [ + "person_clone_index", + "benunit_clone_index", + "household_clone_index", + "source_household_id", + "source_year", + "source_household_key" + ], + "rewrites": [], + "source": "Microcosm structural operator (microcosm#931); the FRS spine's own households, no publisher input", + "stage": "geographic_expand", + "survey": "Family Resources Survey 2024-25" + }, + "geographic_identity": { + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "frs_release.json", + "role": "frs_release", + "runtime_sha256_required": true + } + ], + "grain": "household", + "nonnegative_outputs": [], + "notes": "Keys every household for the area assignment (microcosm#931): geography_household_key, as the dense line's uk.full.identity writes it, from the identity source (source), the FRS vintage frs_release.json pins for the household's source_year, its source_household_id and its clone path, read from the lineage columns geographic_expand writes, never from ids, weights or values: no branch for clone 0 and the geographic_support branch at ordinal k for clone k. So clone 0 keys as the dense line keys the same base household, and growing K never changes an existing household's key. A source year the release pins no vintage for, and keys that repeat, are refused by count. The copy steps after it copy a household's key with the household. Its evidence gives the distinct source households rounded by the shared disclosure rules, that the keys are unique, the source, the vintages and the columns it reads; no key is printed.", + "operations": [ + { + "kind": "key_households_by_clone_path", + "source": "frs", + "vintage_resource": "frs_release.json" + } + ], + "outputs": [ + "geography_household_key" + ], + "rewrites": [], + "source": "Microcosm structural operator (microcosm#931); the FRS vintage frs_release.json pins, no publisher input", + "stage": "geographic_identity", + "survey": "Family Resources Survey 2024-25" + }, "high_value_home_support": { "artifacts": [], "grain": "household", @@ -3089,7 +3168,7 @@ "copies": 6, "copies_column": "high_value_support_copies", "copy_index_column": "high_value_support_copy_index", - "copy_rule": "every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", + "copy_rule": "every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", "declared_factor": 1, "flag_column": "household_is_high_value_support_copy", "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", @@ -4408,7 +4487,7 @@ "stocks_and_shares_isa_subscriptions", "lifetime_isa_payments" ], - "notes": "Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household's was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7's income bands, Table 9.8's age and sex and Table 9.9's regions and selected by keyed walks, the both class first. Amounts follow Table 9.7's subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person's amount by their balance shares, shifted so the class's stocks-and-shares total is Table 9.4's 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. People grow to 2024-25 by Table 9.4's account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4's 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7's last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (Mar\u00eda's rulings of 29 October 2026): Table 9.7's cash bands sit above Table 9.4, and growing that excess by Table 9.4's ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4's 2024-25 Lifetime ISA amount. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.", + "notes": "Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household's was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7's income bands, Table 9.8's age and sex and Table 9.9's regions and selected by keyed walks, the both class first. Amounts follow Table 9.7's subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person's amount by their balance shares, shifted so the class's stocks-and-shares total is Table 9.4's 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. Amounts go in the order of a key that blends each subscriber's weighted rank on the class's balance (cash, stocks and shares, or the two together) with a keyed uniform at balance_rank_weight; tied balances share one weighted mid-rank, so the keyed uniform alone orders them. People grow to 2024-25 by Table 9.4's account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4's 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7's last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (Mar\u00eda's rulings of 29 October 2026): Table 9.7's cash bands sit above Table 9.4, and growing that excess by Table 9.4's ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4's 2024-25 Lifetime ISA amount and ordered by the same blend, on the Lifetime ISA balance over every holder aged 18 to 47, with tied balances again sharing one mid-rank; the Lifetime ISA operation declares no rank weight and reuses the amounts operation's balance_rank_weight. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.", "operations": [ { "artifact_role": "isa_statistics_facts", @@ -6572,7 +6651,7 @@ ], "grain": "person", "nonnegative_outputs": [], - "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine's attendance once, at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula's last step over every person in the frame, from the engine's own values at the calibration year, each decided by a household's own records, and its own parameters: household market income plus benefits less the members' income tax and National Insurance, each household's weight once per member, microdf's weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine before its geographic clones exist), the stage stores the engine's flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine's flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.", + "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage computes the formula's attendance once, over the whole frame at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula's last step over every person in the frame, from the engine's own values at the calibration year, each decided by a household's own records, and its own parameters: household market income plus benefits less the members' income tax and National Insurance, each household's weight once per member, microdf's weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine of one geographic clone), the stage stores the engine's flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine's flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.", "operations": [ { "draw": "attends_private_school_random_draw", @@ -6593,7 +6672,7 @@ "attends_private_school" ], "rewrites": [], - "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", + "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor) over every person in the frame, from the pinned engine's values and parameters", "stage": "private_school_attendance", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" }, @@ -6961,7 +7040,7 @@ "employment_income", "salary_sacrifice_pre_conversion_pay" ], - "notes": "The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household's members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin's answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR's 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (Mar\u00eda's ruling of 2026-10-24 on Vahid's review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult's FRS bonus is its twin's, beside the tape's pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell instead.", + "notes": "The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household's members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin's answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR's 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (Mar\u00eda's ruling of 2026-10-24 on Vahid's review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult's FRS bonus is its twin's, beside the tape's pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell, in its own geographic clone, instead.", "operations": [ { "artifact_role": "obr_salary_sacrifice_costing_facts", @@ -7028,7 +7107,7 @@ "seed": 2024, "solve": "the candidates in the order of their identity-keyed uniforms, taken while the projected base is nearer the anchor's OBR base: the anchor's weight growth times the sum of person weight times (max(0, g (S + B) - threshold) - max(0, g S - threshold)), with g the anchor's earnings growth, S the salary sacrifice and B the bonus sacrificed; every candidate where even all fall short", "source": "frs_bonus_amount", - "spi_bonus_draw": "an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell, by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below", + "spi_bonus_draw": "an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell and the same geographic clone (household_clone_index; a spine without it is one clone), by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles over every clone at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below, and the lowest the one above", "spi_cell_quantiles": [ 0.1, 0.2, @@ -7400,7 +7479,7 @@ "nonnegative_outputs": [ "spi_income_band_donor_lower_bound" ], - "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, with the seats counted in distinct source households, each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves).", + "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, and every clone takes the same seats, sized against the eligible households counted in distinct source households. Each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves).", "operations": [ { "band_column": "spi_income_band_donor_lower_bound", @@ -7417,7 +7496,7 @@ "conservation": "exact_total", "count_rule": "max(minimum_donors_per_band, ceil(published band taxpayers / maximum_donor_weight))", "declared_factor": 1, - "draw": "weighted_without_replacement_identity_keyed within each geographic clone (one before uk.full.expand): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", + "draw": "weighted_without_replacement_identity_keyed within each geographic clone (a spine without household_clone_index is one clone): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", "expected_band_count": 4, "flag_column": "household_is_spi_income_band_donor", "funding": "stratum reallocation per geographic clone: every incumbent household of the stratum a donor's source household belongs to, in the donor's clone, is scaled by 1 - donor mass / stratum mass", @@ -7443,7 +7522,7 @@ "taxpayer_count_period": "build_period_tax_year", "taxpayer_count_resource": "hmrc_itl_taxpayer_counts.json", "weight_kind_out": "importance", - "weight_rule": "published band taxpayers / (geographic clone count x band donor count), equal within a band" + "weight_rule": "published band taxpayers / (geographic clone count x one clone's band donor count), equal within a band" } ], "outputs": [ @@ -7476,11 +7555,9 @@ "benunit_support_clone_index", "benunit_source_id", "household_support_clone_index", - "household_source_id", - "source_household_id", - "source_year" + "household_source_id" ], - "notes": "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's draw depends on its own id alone. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069).", + "notes": "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's uniform depends on its own id and the stage's seed and salt alone, and whether it is copied also on that probability. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069).", "operations": [ { "channels": { @@ -7535,13 +7612,10 @@ "benunit_source_id", "household_support_channel", "household_support_clone_index", - "household_source_id", - "source_household_id", - "source_year", - "source_household_key" + "household_source_id" ], "rewrites": [], - "source": "Synthetic SPI support channel drawn from the raw FRS spine as a keyed Bernoulli share, each household on its own identity-keyed uniform.", + "source": "Synthetic SPI support channel drawn from every geographic clone of the raw FRS spine's households as a keyed Bernoulli share, each household on its own identity-keyed uniform.", "stage": "spi_support_channel", "survey": "Family Resources Survey 2024-25" }, @@ -7714,7 +7788,7 @@ "artifacts": [], "grain": "benunit", "nonnegative_outputs": [], - "notes": "Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donors widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (Mar\u00eda's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base.", + "notes": "Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donor survey records (a unit's geographic clones count once, every other copy as a record of its own) widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.174.1 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (Mar\u00eda's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base.", "operations": [ { "aggregates": { diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/frs_raw/househol.tab b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/frs_raw/househol.tab index 6ba4eb8ba0..98dc95cc73 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/frs_raw/househol.tab +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/frs_raw/househol.tab @@ -1,136 +1,136 @@ -SERNUM GROSS4 GVTREGNO PTENTYP2 TYPEACC BEDROOM6 CTANNUAL CTPAID CTBAND CTDISC CT25D50D CTREB CTREBAMT ADULTH HRPNUM CSEWAMT CWATAMTD CWATAMT1 CSEWAMT1 WATSEWRT NIRATLIA RT2REBAM HHRENT SUBRENT TENTYP2 MORTINT STRUINS CHRGAMT1 CHRGAMT2 CHRGAMT3 CHRGAMT4 CHRGAMT5 CHRGAMT6 CHRGAMT7 CHRGAMT8 CHRGAMT9 INTDATE HHSTAT -1 1.01 11 6 2 2 910.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23665.0 1 -2 1.02 12 5 3 3 920.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23666.0 1 -3 1.03 1 6 4 4 930.0 1 4 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23667.0 1 -4 1.04 8 5 1 1 940.0 1 5 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23668.0 1 -5 1.05 11 6 2 2 950.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23669.0 1 -6 1.06 12 5 3 3 960.0 1 7 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23670.0 1 -7 1.07 1 6 4 4 970.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23671.0 1 -8 1.08 8 5 1 1 980.0 1 2 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23672.0 1 -9 1.09 11 6 2 2 990.0 1 3 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23673.0 1 -10 1.1 12 5 3 3 1000.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23674.0 1 -11 1.11 1 6 4 4 1010.0 1 5 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23675.0 2 -12 1.12 8 5 1 1 0.0 1 6 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23676.0 1 -13 1.13 11 6 2 2 1030.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23677.0 1 -14 1.1400000000000001 12 5 3 3 1040.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23678.0 1 -15 1.15 1 6 4 4 1050.0 1 2 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23679.0 1 -16 1.16 8 5 1 1 1060.0 1 3 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23680.0 1 -17 1.17 11 6 2 2 1070.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23681.0 1 -18 1.18 12 5 3 3 1080.0 1 5 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23682.0 1 -19 1.19 1 6 4 4 1090.0 1 6 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23683.0 1 -20 1.2 8 5 1 1 1100.0 1 7 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23684.0 1 -21 1.21 11 6 2 2 1110.0 1 1 1 1 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23685.0 1 -22 1.22 12 5 3 3 1120.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23686.0 2 -23 1.23 1 6 4 4 1130.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23687.0 1 -24 1.24 8 5 1 1 1140.0 1 4 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23688.0 1 -25 1.25 11 6 2 2 1150.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23689.0 1 -26 1.26 12 5 3 3 1160.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23690.0 1 -27 1.27 1 6 4 4 1170.0 1 7 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23691.0 1 -28 1.28 8 5 1 1 1180.0 1 1 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23692.0 1 -29 1.29 11 6 2 2 1190.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23693.0 1 -30 1.3 12 5 3 3 1200.0 1 3 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23694.0 1 -31 1.31 1 6 4 4 1210.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23695.0 1 -32 1.32 8 5 1 1 1220.0 1 5 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23696.0 1 -33 1.33 11 6 2 2 1230.0 1 6 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23697.0 2 -34 1.34 12 5 3 3 1240.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23698.0 1 -35 1.35 1 6 4 4 1250.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23699.0 1 -36 1.3599999999999999 8 5 1 1 1260.0 1 2 1 2 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23700.0 1 -37 1.37 11 6 2 2 1270.0 1 3 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23701.0 1 -38 1.38 12 5 3 3 1280.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23702.0 1 -39 1.3900000000000001 1 6 4 4 1290.0 1 5 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23703.0 1 -40 1.4 8 5 1 1 1300.0 1 6 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23704.0 1 -41 1.41 11 6 2 2 1310.0 1 7 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23705.0 1 -42 1.42 12 5 3 3 1320.0 1 1 1 1 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23706.0 1 -43 1.43 1 6 4 4 1330.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23707.0 1 -44 1.44 8 5 1 1 1340.0 1 3 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23708.0 2 -45 1.45 11 6 2 2 1350.0 1 4 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23709.0 1 -46 1.46 12 5 3 3 1360.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23710.0 1 -47 1.47 1 6 4 4 1370.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23711.0 1 -48 1.48 8 5 1 1 1380.0 1 7 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23712.0 1 -49 1.49 11 6 2 2 1390.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23713.0 1 -50 1.5 12 5 3 3 1400.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23714.0 1 -51 1.51 1 6 4 4 1410.0 1 3 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23715.0 1 -52 1.52 8 5 1 1 1420.0 1 4 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23716.0 1 -53 1.53 11 6 2 2 1430.0 1 5 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23717.0 1 -54 1.54 12 5 3 3 1440.0 1 6 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23718.0 1 -55 1.55 1 6 4 4 1450.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23719.0 2 -56 1.56 8 5 1 1 1460.0 1 1 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23720.0 1 -57 1.5699999999999998 11 6 2 2 1470.0 1 2 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23721.0 1 -58 1.58 12 5 3 3 1480.0 1 3 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23722.0 1 -59 1.5899999999999999 1 6 4 4 1490.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23723.0 1 -60 1.6 8 5 1 1 1500.0 1 5 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23724.0 1 -61 1.6099999999999999 11 6 2 2 1510.0 1 6 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23725.0 1 -62 1.62 12 5 3 3 1520.0 1 7 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23726.0 1 -63 1.63 1 6 4 4 1530.0 1 1 1 2 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23727.0 1 -64 1.6400000000000001 8 5 1 1 1540.0 1 2 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23728.0 1 -65 1.65 11 6 2 2 1550.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23729.0 1 -66 1.6600000000000001 12 5 3 3 1560.0 1 4 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23730.0 2 -67 1.67 1 6 4 4 1570.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23731.0 1 -68 1.6800000000000002 8 5 1 1 1580.0 1 6 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23732.0 1 -69 1.69 11 6 2 2 1590.0 1 7 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23733.0 1 -70 1.7 12 5 3 3 1600.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23734.0 1 -71 1.71 1 6 4 4 1610.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23735.0 1 -72 1.72 8 5 1 1 1620.0 1 3 1 2 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23736.0 1 -73 1.73 11 6 2 2 1630.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23737.0 1 -74 1.74 12 5 3 3 1640.0 1 5 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23738.0 1 -75 1.75 1 6 4 4 1650.0 1 6 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23739.0 1 -76 1.76 8 5 1 1 1660.0 1 7 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23740.0 1 -77 1.77 11 6 2 2 1670.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23741.0 2 -78 1.78 12 5 3 3 1680.0 1 2 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23742.0 1 -79 1.79 1 6 4 4 1690.0 1 3 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23743.0 1 -80 1.8 8 5 1 1 1700.0 1 4 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23744.0 1 -81 1.81 11 6 2 2 1710.0 1 5 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23745.0 1 -82 1.8199999999999998 12 5 3 3 1720.0 1 6 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23746.0 1 -83 1.83 1 6 4 4 1730.0 1 7 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23747.0 1 -84 1.8399999999999999 8 5 1 1 1740.0 1 1 1 1 1 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23748.0 1 -85 1.85 11 6 2 2 1750.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23749.0 1 -86 1.8599999999999999 12 5 3 3 1760.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23750.0 1 -87 1.87 1 6 4 4 1770.0 1 4 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23751.0 1 -88 1.88 8 5 1 1 1780.0 1 5 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23752.0 2 -89 1.8900000000000001 11 6 2 2 1790.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23753.0 1 -90 1.9 12 5 3 3 1800.0 1 7 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23754.0 1 -91 1.9100000000000001 1 6 4 4 1810.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23755.0 1 -92 1.92 8 5 1 1 1820.0 1 2 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23756.0 1 -93 1.9300000000000002 11 6 2 2 1830.0 1 3 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23757.0 1 -94 1.94 12 5 3 3 1840.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23758.0 1 -95 1.95 1 6 4 4 1850.0 1 5 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23759.0 1 -96 1.96 8 5 1 1 1860.0 1 6 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23760.0 1 -97 1.97 11 6 2 2 1870.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23761.0 1 -98 1.98 12 5 3 3 1880.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23762.0 1 -99 1.99 1 6 4 4 1890.0 1 2 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23763.0 2 -100 2.0 8 5 1 1 1900.0 1 3 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23764.0 1 -101 2.01 11 6 2 2 1910.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23765.0 1 -102 2.02 12 5 3 3 1920.0 1 5 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23766.0 1 -103 2.0300000000000002 1 6 4 4 1930.0 1 6 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23767.0 1 -104 2.04 8 5 1 1 1940.0 1 7 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23768.0 1 -105 2.05 11 6 2 2 1950.0 1 1 1 1 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23769.0 1 -106 2.06 12 5 3 3 1960.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23770.0 1 -107 2.0700000000000003 1 6 4 4 1970.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23771.0 1 -108 2.08 8 5 1 1 1980.0 1 4 1 2 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23772.0 1 -109 2.09 11 6 2 2 1990.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23773.0 1 -110 2.1 12 5 3 3 2000.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23774.0 2 -111 2.1100000000000003 1 6 4 4 2010.0 1 7 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23775.0 1 -112 2.12 8 5 1 1 2020.0 1 1 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23776.0 1 -113 2.13 11 6 2 2 2030.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23777.0 1 -114 2.1399999999999997 12 5 3 3 2040.0 1 3 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23778.0 1 -115 2.15 1 6 4 4 2050.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23779.0 1 -116 2.16 8 5 1 1 2060.0 1 5 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23780.0 1 -117 2.17 11 6 2 2 2070.0 1 6 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23781.0 1 -118 2.1799999999999997 12 5 3 3 2080.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23782.0 1 -119 2.19 1 6 4 4 2090.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23783.0 1 -120 2.2 8 5 1 1 2100.0 1 2 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23664.0 1 -121 2.21 11 6 2 2 2110.0 1 3 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23665.0 2 -122 2.2199999999999998 12 5 3 3 2120.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23666.0 1 -123 2.23 1 6 4 4 2130.0 1 5 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23667.0 1 -124 2.24 8 5 1 1 2140.0 1 6 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23668.0 1 -125 2.25 11 6 2 2 2150.0 1 7 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23669.0 1 -126 2.26 12 5 3 3 2160.0 1 1 1 2 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23670.0 1 -127 2.27 1 6 4 4 2170.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23671.0 1 -128 2.2800000000000002 8 5 1 1 2180.0 1 3 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23672.0 1 -129 2.29 11 6 2 2 2190.0 1 4 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23673.0 1 -130 2.3 12 5 3 3 2200.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23674.0 1 -131 2.31 1 6 4 4 2210.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23675.0 1 -132 2.3200000000000003 8 5 1 1 2220.0 1 7 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23676.0 2 -133 2.33 11 6 2 2 2230.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23677.0 1 -134 2.34 12 5 3 3 2240.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23678.0 1 -135 2.35 1 6 4 4 2250.0 1 3 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23679.0 1 +SERNUM GROSS4 GVTREGNO IMDE IMDW IMDS IMDN PTENTYP2 TYPEACC BEDROOM6 CTANNUAL CTPAID CTBAND CTDISC CT25D50D CTREB CTREBAMT ADULTH HRPNUM CSEWAMT CWATAMTD CWATAMT1 CSEWAMT1 WATSEWRT NIRATLIA RT2REBAM HHRENT SUBRENT TENTYP2 MORTINT STRUINS CHRGAMT1 CHRGAMT2 CHRGAMT3 CHRGAMT4 CHRGAMT5 CHRGAMT6 CHRGAMT7 CHRGAMT8 CHRGAMT9 INTDATE HHSTAT +1 1.01 11 1 6 2 2 910.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23665.0 1 +2 1.02 12 1 5 3 3 920.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23666.0 1 +3 1.03 1 1 6 4 4 930.0 1 4 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23667.0 1 +4 1.04 8 2 5 1 1 940.0 1 5 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23668.0 1 +5 1.05 11 2 6 2 2 950.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23669.0 1 +6 1.06 12 2 5 3 3 960.0 1 7 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23670.0 1 +7 1.07 1 2 6 4 4 970.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23671.0 1 +8 1.08 8 3 5 1 1 980.0 1 2 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23672.0 1 +9 1.09 11 3 6 2 2 990.0 1 3 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23673.0 1 +10 1.1 12 3 5 3 3 1000.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23674.0 1 +11 1.11 1 3 6 4 4 1010.0 1 5 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23675.0 2 +12 1.12 8 4 5 1 1 0.0 1 6 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23676.0 1 +13 1.13 11 4 6 2 2 1030.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23677.0 1 +14 1.1400000000000001 12 4 5 3 3 1040.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23678.0 1 +15 1.15 1 4 6 4 4 1050.0 1 2 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23679.0 1 +16 1.16 8 5 5 1 1 1060.0 1 3 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23680.0 1 +17 1.17 11 6 2 2 1070.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23681.0 1 +18 1.18 12 5 5 3 3 1080.0 1 5 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23682.0 1 +19 1.19 1 5 6 4 4 1090.0 1 6 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23683.0 1 +20 1.2 8 6 5 1 1 1100.0 1 7 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23684.0 1 +21 1.21 11 6 6 2 2 1110.0 1 1 1 1 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23685.0 1 +22 1.22 12 6 5 3 3 1120.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23686.0 2 +23 1.23 1 6 6 4 4 1130.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23687.0 1 +24 1.24 8 7 5 1 1 1140.0 1 4 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23688.0 1 +25 1.25 11 7 6 2 2 1150.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23689.0 1 +26 1.26 12 7 5 3 3 1160.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23690.0 1 +27 1.27 1 7 6 4 4 1170.0 1 7 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23691.0 1 +28 1.28 8 8 5 1 1 1180.0 1 1 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23692.0 1 +29 1.29 11 8 6 2 2 1190.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23693.0 1 +30 1.3 12 8 5 3 3 1200.0 1 3 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23694.0 1 +31 1.31 1 8 6 4 4 1210.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23695.0 1 +32 1.32 8 9 5 1 1 1220.0 1 5 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23696.0 1 +33 1.33 11 9 6 2 2 1230.0 1 6 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23697.0 2 +34 1.34 12 5 3 3 1240.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23698.0 1 +35 1.35 1 9 6 4 4 1250.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23699.0 1 +36 1.3599999999999999 8 10 5 1 1 1260.0 1 2 1 2 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23700.0 1 +37 1.37 11 10 6 2 2 1270.0 1 3 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23701.0 1 +38 1.38 12 10 5 3 3 1280.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23702.0 1 +39 1.3900000000000001 1 10 6 4 4 1290.0 1 5 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23703.0 1 +40 1.4 8 1 5 1 1 1300.0 1 6 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23704.0 1 +41 1.41 11 1 6 2 2 1310.0 1 7 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23705.0 1 +42 1.42 12 1 5 3 3 1320.0 1 1 1 1 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23706.0 1 +43 1.43 1 1 6 4 4 1330.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23707.0 1 +44 1.44 8 2 5 1 1 1340.0 1 3 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23708.0 2 +45 1.45 11 2 6 2 2 1350.0 1 4 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23709.0 1 +46 1.46 12 2 5 3 3 1360.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23710.0 1 +47 1.47 1 2 6 4 4 1370.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23711.0 1 +48 1.48 8 3 5 1 1 1380.0 1 7 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23712.0 1 +49 1.49 11 3 6 2 2 1390.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23713.0 1 +50 1.5 12 3 5 3 3 1400.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23714.0 1 +51 1.51 1 6 4 4 1410.0 1 3 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23715.0 1 +52 1.52 8 4 5 1 1 1420.0 1 4 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23716.0 1 +53 1.53 11 4 6 2 2 1430.0 1 5 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23717.0 1 +54 1.54 12 4 5 3 3 1440.0 1 6 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23718.0 1 +55 1.55 1 4 6 4 4 1450.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23719.0 2 +56 1.56 8 5 5 1 1 1460.0 1 1 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23720.0 1 +57 1.5699999999999998 11 5 6 2 2 1470.0 1 2 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23721.0 1 +58 1.58 12 5 5 3 3 1480.0 1 3 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23722.0 1 +59 1.5899999999999999 1 5 6 4 4 1490.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23723.0 1 +60 1.6 8 6 5 1 1 1500.0 1 5 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23724.0 1 +61 1.6099999999999999 11 6 6 2 2 1510.0 1 6 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23725.0 1 +62 1.62 12 6 5 3 3 1520.0 1 7 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23726.0 1 +63 1.63 1 6 6 4 4 1530.0 1 1 1 2 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23727.0 1 +64 1.6400000000000001 8 7 5 1 1 1540.0 1 2 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23728.0 1 +65 1.65 11 7 6 2 2 1550.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23729.0 1 +66 1.6600000000000001 12 7 5 3 3 1560.0 1 4 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23730.0 2 +67 1.67 1 7 6 4 4 1570.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23731.0 1 +68 1.6800000000000002 8 5 1 1 1580.0 1 6 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23732.0 1 +69 1.69 11 8 6 2 2 1590.0 1 7 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23733.0 1 +70 1.7 12 8 5 3 3 1600.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23734.0 1 +71 1.71 1 8 6 4 4 1610.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23735.0 1 +72 1.72 8 9 5 1 1 1620.0 1 3 1 2 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23736.0 1 +73 1.73 11 9 6 2 2 1630.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23737.0 1 +74 1.74 12 9 5 3 3 1640.0 1 5 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23738.0 1 +75 1.75 1 9 6 4 4 1650.0 1 6 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23739.0 1 +76 1.76 8 10 5 1 1 1660.0 1 7 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23740.0 1 +77 1.77 11 10 6 2 2 1670.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23741.0 2 +78 1.78 12 10 5 3 3 1680.0 1 2 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23742.0 1 +79 1.79 1 10 6 4 4 1690.0 1 3 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23743.0 1 +80 1.8 8 1 5 1 1 1700.0 1 4 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23744.0 1 +81 1.81 11 1 6 2 2 1710.0 1 5 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23745.0 1 +82 1.8199999999999998 12 1 5 3 3 1720.0 1 6 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23746.0 1 +83 1.83 1 1 6 4 4 1730.0 1 7 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23747.0 1 +84 1.8399999999999999 8 2 5 1 1 1740.0 1 1 1 1 1 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23748.0 1 +85 1.85 11 6 2 2 1750.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23749.0 1 +86 1.8599999999999999 12 2 5 3 3 1760.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23750.0 1 +87 1.87 1 2 6 4 4 1770.0 1 4 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23751.0 1 +88 1.88 8 3 5 1 1 1780.0 1 5 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23752.0 2 +89 1.8900000000000001 11 3 6 2 2 1790.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23753.0 1 +90 1.9 12 3 5 3 3 1800.0 1 7 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23754.0 1 +91 1.9100000000000001 1 3 6 4 4 1810.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23755.0 1 +92 1.92 8 4 5 1 1 1820.0 1 2 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23756.0 1 +93 1.9300000000000002 11 4 6 2 2 1830.0 1 3 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23757.0 1 +94 1.94 12 4 5 3 3 1840.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23758.0 1 +95 1.95 1 4 6 4 4 1850.0 1 5 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23759.0 1 +96 1.96 8 5 5 1 1 1860.0 1 6 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23760.0 1 +97 1.97 11 5 6 2 2 1870.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23761.0 1 +98 1.98 12 5 5 3 3 1880.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23762.0 1 +99 1.99 1 5 6 4 4 1890.0 1 2 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23763.0 2 +100 2.0 8 6 5 1 1 1900.0 1 3 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23764.0 1 +101 2.01 11 6 6 2 2 1910.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23765.0 1 +102 2.02 12 5 3 3 1920.0 1 5 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23766.0 1 +103 2.0300000000000002 1 6 6 4 4 1930.0 1 6 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23767.0 1 +104 2.04 8 7 5 1 1 1940.0 1 7 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23768.0 1 +105 2.05 11 7 6 2 2 1950.0 1 1 1 1 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23769.0 1 +106 2.06 12 7 5 3 3 1960.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23770.0 1 +107 2.0700000000000003 1 7 6 4 4 1970.0 1 3 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23771.0 1 +108 2.08 8 8 5 1 1 1980.0 1 4 1 2 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23772.0 1 +109 2.09 11 8 6 2 2 1990.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23773.0 1 +110 2.1 12 8 5 3 3 2000.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23774.0 2 +111 2.1100000000000003 1 8 6 4 4 2010.0 1 7 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23775.0 1 +112 2.12 8 9 5 1 1 2020.0 1 1 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23776.0 1 +113 2.13 11 9 6 2 2 2030.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23777.0 1 +114 2.1399999999999997 12 9 5 3 3 2040.0 1 3 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23778.0 1 +115 2.15 1 9 6 4 4 2050.0 1 4 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23779.0 1 +116 2.16 8 10 5 1 1 2060.0 1 5 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23780.0 1 +117 2.17 11 10 6 2 2 2070.0 1 6 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23781.0 1 +118 2.1799999999999997 12 10 5 3 3 2080.0 1 7 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23782.0 1 +119 2.19 1 6 4 4 2090.0 1 1 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23783.0 1 +120 2.2 8 1 5 1 1 2100.0 1 2 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23664.0 1 +121 2.21 11 1 6 2 2 2110.0 1 3 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23665.0 2 +122 2.2199999999999998 12 1 5 3 3 2120.0 1 4 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23666.0 1 +123 2.23 1 1 6 4 4 2130.0 1 5 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23667.0 1 +124 2.24 8 2 5 1 1 2140.0 1 6 2 1 1.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23668.0 1 +125 2.25 11 2 6 2 2 2150.0 1 7 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23669.0 1 +126 2.26 12 2 5 3 3 2160.0 1 1 1 2 1 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23670.0 1 +127 2.27 1 2 6 4 4 2170.0 1 2 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23671.0 1 +128 2.2800000000000002 8 3 5 1 1 2180.0 1 3 2 1 2.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23672.0 1 +129 2.29 11 3 6 2 2 2190.0 1 4 1 1 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23673.0 1 +130 2.3 12 3 5 3 3 2200.0 1 5 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23674.0 1 +131 2.31 1 3 6 4 4 2210.0 1 6 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23675.0 1 +132 2.3200000000000003 8 4 5 1 1 2220.0 1 7 1 1 2 0.0 1 1 0.0 4.0 5.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23676.0 2 +133 2.33 11 4 6 2 2 2230.0 1 1 2 1 1.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23677.0 1 +134 2.34 12 4 5 3 3 2240.0 1 2 2 1 2.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23678.0 1 +135 2.35 1 4 6 4 4 2250.0 1 3 1 2 2 0.0 1 1 0.0 3.0 0.0 0.0 5.0 0.0 5 7.0 8.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 23679.0 1 diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/household.csv b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/household.csv index 58abcd1261..f68b74940b 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/household.csv +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/household.csv @@ -1,136 +1,136 @@ -household_id,region,tenure_type,accommodation_type,num_bedrooms,council_tax_reported,council_tax_band,council_tax_rebate,council_tax_single_adult_raw,water_and_sewerage_charges,domestic_rates,rent,subrent,mortgage_interest_repayment,mortgage_capital_repayment,structural_insurance_payments,housing_service_charges,external_child_payments -1,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,9.1_0000000000000000e+02,B,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -2,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,9.2_0000000000000000e+02,C,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -3,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,9.3_0000000000000000e+02,D,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -4,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,9.4_0000000000000000e+02,E,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -5,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,9.5_0000000000000000e+02,F,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -6,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,9.6_0000000000000000e+02,G,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -7,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,9.7_0000000000000000e+02,A,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -8,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,9.8_0000000000000000e+02,B,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -9,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,9.9_0000000000000000e+02,C,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -10,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.0_0000000000000000e+03,D,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -11,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.0_1000000000000000e+03,E,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -12,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,0.0_0000000000000000e+00,F,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -13,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.0_3000000000000000e+03,G,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -14,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.0_4000000000000000e+03,A,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -15,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.0_5000000000000000e+03,B,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -16,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.0_6000000000000000e+03,C,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -17,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.0_7000000000000000e+03,D,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -18,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.0_8000000000000000e+03,E,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -19,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.0_9000000000000000e+03,F,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -20,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.1_0000000000000000e+03,G,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -21,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.1_1000000000000000e+03,A,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -22,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.1_2000000000000000e+03,B,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -23,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.1_3000000000000000e+03,C,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -24,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.1_4000000000000000e+03,D,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -25,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.1_5000000000000000e+03,E,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -26,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.1_6000000000000000e+03,F,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -27,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.1_7000000000000000e+03,G,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -28,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.1_8000000000000000e+03,A,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -29,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.1_9000000000000000e+03,B,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -30,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.2_0000000000000000e+03,C,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -31,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.2_1000000000000000e+03,D,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -32,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.2_2000000000000000e+03,E,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -33,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.2_3000000000000000e+03,F,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -34,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.2_4000000000000000e+03,G,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -35,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.2_5000000000000000e+03,A,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -36,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.2_6000000000000000e+03,B,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -37,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.2_7000000000000000e+03,C,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -38,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.2_8000000000000000e+03,D,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -39,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.2_9000000000000000e+03,E,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -40,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.3_0000000000000000e+03,F,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -41,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.3_1000000000000000e+03,G,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -42,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.3_2000000000000000e+03,A,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -43,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.3_3000000000000000e+03,B,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -44,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.3_4000000000000000e+03,C,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -45,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.3_5000000000000000e+03,D,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -46,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.3_6000000000000000e+03,E,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -47,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.3_7000000000000000e+03,F,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -48,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.3_8000000000000000e+03,G,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -49,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.3_9000000000000000e+03,A,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -50,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.4_0000000000000000e+03,B,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -51,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.4_1000000000000000e+03,C,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -52,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.4_2000000000000000e+03,D,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -53,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.4_3000000000000000e+03,E,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -54,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.4_4000000000000000e+03,F,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -55,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.4_5000000000000000e+03,G,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -56,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.4_6000000000000000e+03,A,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -57,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.4_7000000000000000e+03,B,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -58,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.4_8000000000000000e+03,C,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -59,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.4_9000000000000000e+03,D,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -60,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.5_0000000000000000e+03,E,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -61,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.5_1000000000000000e+03,F,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -62,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.5_2000000000000000e+03,G,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -63,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.5_3000000000000000e+03,A,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -64,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.5_4000000000000000e+03,B,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -65,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.5_5000000000000000e+03,C,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -66,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.5_6000000000000000e+03,D,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -67,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.5_7000000000000000e+03,E,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -68,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.5_8000000000000000e+03,F,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -69,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.5_9000000000000000e+03,G,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -70,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.6_0000000000000000e+03,A,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -71,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.6_1000000000000000e+03,B,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -72,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.6_2000000000000000e+03,C,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -73,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.6_3000000000000000e+03,D,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -74,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.6_4000000000000000e+03,E,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -75,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.6_5000000000000000e+03,F,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -76,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.6_6000000000000000e+03,G,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -77,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.6_7000000000000000e+03,A,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -78,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.6_8000000000000000e+03,B,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -79,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.6_9000000000000000e+03,C,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -80,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.7_0000000000000000e+03,D,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -81,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.7_1000000000000000e+03,E,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -82,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.7_2000000000000000e+03,F,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -83,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.7_3000000000000000e+03,G,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -84,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.7_4000000000000000e+03,A,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -85,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.7_5000000000000000e+03,B,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -86,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.7_6000000000000000e+03,C,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -87,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.7_7000000000000000e+03,D,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -88,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.7_8000000000000000e+03,E,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -89,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.7_9000000000000000e+03,F,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -90,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.8_0000000000000000e+03,G,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -91,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.8_1000000000000000e+03,A,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -92,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.8_2000000000000000e+03,B,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -93,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.8_3000000000000000e+03,C,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -94,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.8_4000000000000000e+03,D,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -95,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.8_5000000000000000e+03,E,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -96,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.8_6000000000000000e+03,F,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -97,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.8_7000000000000000e+03,G,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -98,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.8_8000000000000000e+03,A,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -99,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.8_9000000000000000e+03,B,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -100,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.9_0000000000000000e+03,C,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -101,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.9_1000000000000000e+03,D,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -102,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.9_2000000000000000e+03,E,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -103,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.9_3000000000000000e+03,F,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -104,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.9_4000000000000000e+03,G,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -105,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.9_5000000000000000e+03,A,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -106,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.9_6000000000000000e+03,B,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -107,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,1.9_7000000000000000e+03,C,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -108,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.9_8000000000000000e+03,D,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -109,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.9_9000000000000000e+03,E,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -110,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.0_0000000000000000e+03,F,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -111,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,2.0_1000000000000000e+03,G,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -112,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.0_2000000000000000e+03,A,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -113,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.0_3000000000000000e+03,B,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -114,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.0_4000000000000000e+03,C,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -115,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,2.0_5000000000000000e+03,D,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -116,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.0_6000000000000000e+03,E,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -117,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.0_7000000000000000e+03,F,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -118,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.0_8000000000000000e+03,G,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -119,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,2.0_9000000000000000e+03,A,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -120,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.1_0000000000000000e+03,B,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -121,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.1_1000000000000000e+03,C,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -122,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.1_2000000000000000e+03,D,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -123,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,2.1_3000000000000000e+03,E,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -124,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.1_4000000000000000e+03,F,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -125,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.1_5000000000000000e+03,G,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -126,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.1_6000000000000000e+03,A,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -127,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,2.1_7000000000000000e+03,B,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -128,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.1_8000000000000000e+03,C,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -129,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.1_9000000000000000e+03,D,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -130,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.2_0000000000000000e+03,E,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -131,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,2.2_1000000000000000e+03,F,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -132,LONDON,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.2_2000000000000000e+03,G,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -133,WALES,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.2_3000000000000000e+03,A,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -134,SCOTLAND,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.2_4000000000000000e+03,B,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 -135,NORTH_EAST,OWNED_WITH_MORTGAGE,FLAT,4,2.2_5000000000000000e+03,C,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +household_id,region,imd_decile,tenure_type,accommodation_type,num_bedrooms,council_tax_reported,council_tax_band,council_tax_band_observed,council_tax_rebate,council_tax_single_adult_raw,water_and_sewerage_charges,domestic_rates,rent,subrent,mortgage_interest_repayment,mortgage_capital_repayment,structural_insurance_payments,housing_service_charges,external_child_payments +1,WALES,1,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,9.1_0000000000000000e+02,B,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +2,SCOTLAND,1,OWNED_OUTRIGHT,HOUSE_TERRACED,3,9.2_0000000000000000e+02,C,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +3,NORTH_EAST,1,OWNED_WITH_MORTGAGE,FLAT,4,9.3_0000000000000000e+02,D,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +4,LONDON,2,OWNED_OUTRIGHT,HOUSE_DETACHED,1,9.4_0000000000000000e+02,E,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +5,WALES,2,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,9.5_0000000000000000e+02,F,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +6,SCOTLAND,2,OWNED_OUTRIGHT,HOUSE_TERRACED,3,9.6_0000000000000000e+02,G,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +7,NORTH_EAST,2,OWNED_WITH_MORTGAGE,FLAT,4,9.7_0000000000000000e+02,A,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +8,LONDON,3,OWNED_OUTRIGHT,HOUSE_DETACHED,1,9.8_0000000000000000e+02,B,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +9,WALES,3,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,9.9_0000000000000000e+02,C,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +10,SCOTLAND,3,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.0_0000000000000000e+03,D,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +11,NORTH_EAST,3,OWNED_WITH_MORTGAGE,FLAT,4,1.0_1000000000000000e+03,E,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +12,LONDON,4,OWNED_OUTRIGHT,HOUSE_DETACHED,1,0.0_0000000000000000e+00,F,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +13,WALES,4,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.0_3000000000000000e+03,G,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +14,SCOTLAND,4,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.0_4000000000000000e+03,A,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +15,NORTH_EAST,4,OWNED_WITH_MORTGAGE,FLAT,4,1.0_5000000000000000e+03,B,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +16,LONDON,5,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.0_6000000000000000e+03,C,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +17,WALES,0,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.0_7000000000000000e+03,D,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +18,SCOTLAND,5,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.0_8000000000000000e+03,E,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +19,NORTH_EAST,5,OWNED_WITH_MORTGAGE,FLAT,4,1.0_9000000000000000e+03,F,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +20,LONDON,6,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.1_0000000000000000e+03,G,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +21,WALES,6,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.1_1000000000000000e+03,A,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +22,SCOTLAND,6,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.1_2000000000000000e+03,B,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +23,NORTH_EAST,6,OWNED_WITH_MORTGAGE,FLAT,4,1.1_3000000000000000e+03,C,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +24,LONDON,7,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.1_4000000000000000e+03,D,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +25,WALES,7,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.1_5000000000000000e+03,E,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +26,SCOTLAND,7,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.1_6000000000000000e+03,F,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +27,NORTH_EAST,7,OWNED_WITH_MORTGAGE,FLAT,4,1.1_7000000000000000e+03,G,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +28,LONDON,8,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.1_8000000000000000e+03,A,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +29,WALES,8,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.1_9000000000000000e+03,B,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +30,SCOTLAND,8,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.2_0000000000000000e+03,C,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +31,NORTH_EAST,8,OWNED_WITH_MORTGAGE,FLAT,4,1.2_1000000000000000e+03,D,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +32,LONDON,9,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.2_2000000000000000e+03,E,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +33,WALES,9,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.2_3000000000000000e+03,F,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +34,SCOTLAND,0,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.2_4000000000000000e+03,G,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +35,NORTH_EAST,9,OWNED_WITH_MORTGAGE,FLAT,4,1.2_5000000000000000e+03,A,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +36,LONDON,10,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.2_6000000000000000e+03,B,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +37,WALES,10,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.2_7000000000000000e+03,C,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +38,SCOTLAND,10,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.2_8000000000000000e+03,D,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +39,NORTH_EAST,10,OWNED_WITH_MORTGAGE,FLAT,4,1.2_9000000000000000e+03,E,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +40,LONDON,1,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.3_0000000000000000e+03,F,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +41,WALES,1,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.3_1000000000000000e+03,G,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +42,SCOTLAND,1,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.3_2000000000000000e+03,A,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +43,NORTH_EAST,1,OWNED_WITH_MORTGAGE,FLAT,4,1.3_3000000000000000e+03,B,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +44,LONDON,2,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.3_4000000000000000e+03,C,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +45,WALES,2,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.3_5000000000000000e+03,D,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +46,SCOTLAND,2,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.3_6000000000000000e+03,E,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +47,NORTH_EAST,2,OWNED_WITH_MORTGAGE,FLAT,4,1.3_7000000000000000e+03,F,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +48,LONDON,3,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.3_8000000000000000e+03,G,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +49,WALES,3,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.3_9000000000000000e+03,A,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +50,SCOTLAND,3,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.4_0000000000000000e+03,B,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +51,NORTH_EAST,0,OWNED_WITH_MORTGAGE,FLAT,4,1.4_1000000000000000e+03,C,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +52,LONDON,4,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.4_2000000000000000e+03,D,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +53,WALES,4,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.4_3000000000000000e+03,E,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +54,SCOTLAND,4,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.4_4000000000000000e+03,F,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +55,NORTH_EAST,4,OWNED_WITH_MORTGAGE,FLAT,4,1.4_5000000000000000e+03,G,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +56,LONDON,5,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.4_6000000000000000e+03,A,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +57,WALES,5,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.4_7000000000000000e+03,B,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +58,SCOTLAND,5,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.4_8000000000000000e+03,C,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +59,NORTH_EAST,5,OWNED_WITH_MORTGAGE,FLAT,4,1.4_9000000000000000e+03,D,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +60,LONDON,6,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.5_0000000000000000e+03,E,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +61,WALES,6,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.5_1000000000000000e+03,F,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +62,SCOTLAND,6,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.5_2000000000000000e+03,G,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +63,NORTH_EAST,6,OWNED_WITH_MORTGAGE,FLAT,4,1.5_3000000000000000e+03,A,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +64,LONDON,7,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.5_4000000000000000e+03,B,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +65,WALES,7,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.5_5000000000000000e+03,C,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +66,SCOTLAND,7,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.5_6000000000000000e+03,D,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +67,NORTH_EAST,7,OWNED_WITH_MORTGAGE,FLAT,4,1.5_7000000000000000e+03,E,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +68,LONDON,0,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.5_8000000000000000e+03,F,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +69,WALES,8,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.5_9000000000000000e+03,G,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +70,SCOTLAND,8,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.6_0000000000000000e+03,A,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +71,NORTH_EAST,8,OWNED_WITH_MORTGAGE,FLAT,4,1.6_1000000000000000e+03,B,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +72,LONDON,9,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.6_2000000000000000e+03,C,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +73,WALES,9,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.6_3000000000000000e+03,D,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +74,SCOTLAND,9,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.6_4000000000000000e+03,E,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +75,NORTH_EAST,9,OWNED_WITH_MORTGAGE,FLAT,4,1.6_5000000000000000e+03,F,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +76,LONDON,10,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.6_6000000000000000e+03,G,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +77,WALES,10,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.6_7000000000000000e+03,A,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +78,SCOTLAND,10,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.6_8000000000000000e+03,B,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +79,NORTH_EAST,10,OWNED_WITH_MORTGAGE,FLAT,4,1.6_9000000000000000e+03,C,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +80,LONDON,1,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.7_0000000000000000e+03,D,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +81,WALES,1,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.7_1000000000000000e+03,E,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +82,SCOTLAND,1,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.7_2000000000000000e+03,F,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +83,NORTH_EAST,1,OWNED_WITH_MORTGAGE,FLAT,4,1.7_3000000000000000e+03,G,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +84,LONDON,2,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.7_4000000000000000e+03,A,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +85,WALES,0,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.7_5000000000000000e+03,B,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +86,SCOTLAND,2,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.7_6000000000000000e+03,C,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +87,NORTH_EAST,2,OWNED_WITH_MORTGAGE,FLAT,4,1.7_7000000000000000e+03,D,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +88,LONDON,3,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.7_8000000000000000e+03,E,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +89,WALES,3,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.7_9000000000000000e+03,F,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +90,SCOTLAND,3,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.8_0000000000000000e+03,G,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +91,NORTH_EAST,3,OWNED_WITH_MORTGAGE,FLAT,4,1.8_1000000000000000e+03,A,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +92,LONDON,4,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.8_2000000000000000e+03,B,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +93,WALES,4,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.8_3000000000000000e+03,C,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +94,SCOTLAND,4,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.8_4000000000000000e+03,D,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +95,NORTH_EAST,4,OWNED_WITH_MORTGAGE,FLAT,4,1.8_5000000000000000e+03,E,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +96,LONDON,5,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.8_6000000000000000e+03,F,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +97,WALES,5,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.8_7000000000000000e+03,G,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +98,SCOTLAND,5,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.8_8000000000000000e+03,A,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +99,NORTH_EAST,5,OWNED_WITH_MORTGAGE,FLAT,4,1.8_9000000000000000e+03,B,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +100,LONDON,6,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.9_0000000000000000e+03,C,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +101,WALES,6,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.9_1000000000000000e+03,D,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +102,SCOTLAND,0,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.9_2000000000000000e+03,E,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +103,NORTH_EAST,6,OWNED_WITH_MORTGAGE,FLAT,4,1.9_3000000000000000e+03,F,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +104,LONDON,7,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.9_4000000000000000e+03,G,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +105,WALES,7,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.9_5000000000000000e+03,A,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +106,SCOTLAND,7,OWNED_OUTRIGHT,HOUSE_TERRACED,3,1.9_6000000000000000e+03,B,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +107,NORTH_EAST,7,OWNED_WITH_MORTGAGE,FLAT,4,1.9_7000000000000000e+03,C,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +108,LONDON,8,OWNED_OUTRIGHT,HOUSE_DETACHED,1,1.9_8000000000000000e+03,D,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +109,WALES,8,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,1.9_9000000000000000e+03,E,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +110,SCOTLAND,8,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.0_0000000000000000e+03,F,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +111,NORTH_EAST,8,OWNED_WITH_MORTGAGE,FLAT,4,2.0_1000000000000000e+03,G,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +112,LONDON,9,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.0_2000000000000000e+03,A,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +113,WALES,9,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.0_3000000000000000e+03,B,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +114,SCOTLAND,9,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.0_4000000000000000e+03,C,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +115,NORTH_EAST,9,OWNED_WITH_MORTGAGE,FLAT,4,2.0_5000000000000000e+03,D,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +116,LONDON,10,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.0_6000000000000000e+03,E,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +117,WALES,10,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.0_7000000000000000e+03,F,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +118,SCOTLAND,10,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.0_8000000000000000e+03,G,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +119,NORTH_EAST,0,OWNED_WITH_MORTGAGE,FLAT,4,2.0_9000000000000000e+03,A,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +120,LONDON,1,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.1_0000000000000000e+03,B,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +121,WALES,1,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.1_1000000000000000e+03,C,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +122,SCOTLAND,1,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.1_2000000000000000e+03,D,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +123,NORTH_EAST,1,OWNED_WITH_MORTGAGE,FLAT,4,2.1_3000000000000000e+03,E,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +124,LONDON,2,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.1_4000000000000000e+03,F,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +125,WALES,2,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.1_5000000000000000e+03,G,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +126,SCOTLAND,2,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.1_6000000000000000e+03,A,True,0.0_0000000000000000e+00,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +127,NORTH_EAST,2,OWNED_WITH_MORTGAGE,FLAT,4,2.1_7000000000000000e+03,B,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +128,LONDON,3,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.1_8000000000000000e+03,C,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +129,WALES,3,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.1_9000000000000000e+03,D,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +130,SCOTLAND,3,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.2_0000000000000000e+03,E,True,5.2_0000000000000000e+01,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +131,NORTH_EAST,3,OWNED_WITH_MORTGAGE,FLAT,4,2.2_1000000000000000e+03,F,True,1.0_4000000000000000e+02,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +132,LONDON,4,OWNED_OUTRIGHT,HOUSE_DETACHED,1,2.2_2000000000000000e+03,G,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +133,WALES,4,OWNED_WITH_MORTGAGE,HOUSE_SEMI_DETACHED,2,2.2_3000000000000000e+03,A,True,5.2_0000000000000000e+01,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +134,SCOTLAND,4,OWNED_OUTRIGHT,HOUSE_TERRACED,3,2.2_4000000000000000e+03,B,True,1.0_4000000000000000e+02,1,0.0_0000000000000000e+00,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 +135,NORTH_EAST,4,OWNED_WITH_MORTGAGE,FLAT,4,2.2_5000000000000000e+03,C,True,0.0_0000000000000000e+00,1,1.5_6000000000000000e+02,0.0_0000000000000000e+00,2.6_0000000000000000e+02,0.0_0000000000000000e+00,3.6_4000000000000000e+02,1.0_0000000000000000e+01,4.1_6000000000000000e+02,2.3_4000000000000000e+03,1.0_4000000000000000e+02 diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/schema.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/schema.json index df7e779097..c39b81258e 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/schema.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/schema.json @@ -10,6 +10,7 @@ "household": { "accommodation_type": "string", "council_tax_band": "string", + "council_tax_band_observed": "bool", "council_tax_rebate": "float64", "council_tax_reported": "float64", "council_tax_single_adult_raw": "int64", @@ -17,6 +18,7 @@ "external_child_payments": "float64", "household_id": "int64", "housing_service_charges": "float64", + "imd_decile": "int64", "mortgage_capital_repayment": "float64", "mortgage_interest_repayment": "float64", "num_bedrooms": "int64", diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json index 047ed6f476..92a826954e 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/uk_spine.json @@ -1 +1 @@ -{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"879fdd6e1e5832b94af4259b1028f852470bbe2c3ab19b61ce5f92a2e9c12ec4","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","education_grants","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"frs_education_grant_split.boundary","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["brma","region"],"entity":"household","rows":"all"},{"columns":["age","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_brma.checkpoint","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","source_household_id","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_support_channel","stage_contract_sha256":"8a730bdb97c48c48434f346b87a7eb31c169ff511e5850ba19d1682a91e8351b","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.source_household_id","household.source_year","household.source_household_key","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_income_band_donors","stage_contract_sha256":"83c5835361dbe97c3b3f13b38914e269a41ebc16282308d4a0cae53d83128ccf","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","ossben_identifiable_subset","srp_regular_code5","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"high_value_home_support","stage_contract_sha256":"6d8c9503ad786ca285b96914fe9c1d792c7dcf5389841f51bbe295e549283ef8","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_components","stage_contract_sha256":"1e513a7c6644d48ee0952a6d3f3151df3c6a8dd2289bec0b355f40fbbd10475d","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_support_split","stage_contract_sha256":"a874b548dbb238143710da5220fec99e1ba4b69fcabe6bfcbba45de970ac77be","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"1dd10a33849c0377b6e79f5e5077ebad741b8f13de44896f05e7322700d90e3e","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"a0a0d7d311f217d9c8149656efd4b7baf7ee11c2eca2a904de694253ccd171e7","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"salary_sacrifice","stage_contract_sha256":"e6b2fe2c753f50fffd25075b256bcdfb75c1a816112db9a4dd0a61d5f3b54c8b","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","source_household_id","source_year","source_household_key","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"private_school_attendance","stage_contract_sha256":"7e44379c0331fe4c9de05cd76a075c1a6aa22cf0dec6cd2f91b277e70f14a038","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} +{"country":"uk","nodes":[{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Load the source-bound UK FRS root population.","id":"create_uk_frs","inputs":[],"kernel":"uk.create@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"imd_decile","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_band_observed","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"sample_fraction":1.0,"sample_seed":578,"stage_contract_sha256":"010ce0eef3269a13fe1970f72a79150f33761fef1e1dcc02043a2cc903652dd5","time_period":"2024"},"population":null,"sources":["frs"],"structural":"create","weights":null},{"base":"create_uk_frs","citation":"","description":"Ownership boundary for the source-assembling root stage.","id":"frs_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"base":null,"citation":"","description":"Claim the cells assembled by the UK FRS root transform.","id":"frs_spine","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"gender","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"months_since_last_birthday","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"marital_status","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hours_worked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"care_hours","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_care_hours_in_household","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_household_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_benunit_head","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_parent","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_blind","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_uc_claimant","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_claimant_or_partner","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_hbai_dependent_child","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_boarder","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent_paid_as_lodger","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"spousal_maintenance_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"miscellaneous_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_transfer_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"lump_sum_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loan_repayments","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_sick_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_maternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"statutory_paternity_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"student_loans","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"access_fund","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"healthy_start_vouchers","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_breakfasts","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_fruit_veg","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"free_school_meals","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maintenance_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"childcare_expenses","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_asked","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_bonus_amount","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_price_band","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_fuel_code","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_private_fuel","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_company_car_contribution","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"incapacity_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_married","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dependent_children","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"liable_for_share_of_household_rent","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"region","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"imd_decile","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band_observed","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_single_adult_raw","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"external_child_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{},"population":"frs_spine.boundary","sources":[],"structural":"none","weights":null},{"base":"frs_spine.boundary","citation":"","description":"Ownership boundary before age_tail rewrites.","id":"age_tail.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage age_tail.","id":"age_tail","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["external_child_payments","region"],"entity":"household","rows":"all"},{"columns":["gender"],"entity":"person","rows":"all"}],"kernel":"uk.stage.age_tail@1","mass":"conserve","outputs":[{"column":"age","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"age_tail","stage_contract_sha256":"963a34f38caede2525e3b8735f904bb70a1bbef5a272f0eefd4346343444484f","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_relationships.","id":"frs_relationships","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","is_household_head"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_relationships@1","mass":"conserve","outputs":[{"column":"relationship_to_head","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_role","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_family_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_looked_after_by_local_authority","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"child_limit_non_parental_caring_exception","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"ons_household_type","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_relationships","stage_contract_sha256":"ae32bf855753d3b7505e54a18532b1b53a04b6beda764cc09ad09462cf130085","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_employment.","id":"frs_employment","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["ons_household_type","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_employment@1","mass":"conserve","outputs":[{"column":"employment_status","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_employment","stage_contract_sha256":"c4dc3da7179c7b7d6eb67c6f119df1f213e50ee6b83a53660d317126a5773027","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_council_tax.","id":"frs_council_tax","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_council_tax@1","mass":"conserve","outputs":[{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_council_tax","stage_contract_sha256":"06804f54c12d25f2afcb31eec27789802f663cb42689a932dc32f3ba5fc05fff","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_disability.","id":"frs_disability","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax","region"],"entity":"household","rows":"all"},{"columns":["afcs_reported","age","attendance_allowance_reported","dla_m_reported","dla_sc_reported","esa_contrib_reported","esa_income_reported","iidb_reported","incapacity_benefit_reported","pip_dl_reported","pip_m_reported","sda_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_disability@1","mass":"conserve","outputs":[{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_disability","stage_contract_sha256":"ac500b4cb1bc04d4fb72d198592dd376d61da8c150921a6ac53914ac18be0508","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education.","id":"frs_education","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","esa_contrib_reported","esa_income_reported","is_severely_disabled_for_benefits","jsa_contrib_reported","jsa_income_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education@1","mass":"conserve","outputs":[{"column":"current_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"highest_education","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_non_advanced_education","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"is_in_approved_training","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_started_or_accepted_current_education_or_training","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_before_universal_credit_qualifying_young_person_terminal_date","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"adult_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"child_ema","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"reports_outstanding_student_loan","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"age_left_full_time_education","dtype":"int64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_education","stage_contract_sha256":"592ddc77703f19bb3e1d48b6aecffc98b4e2abd3379db84a06e011bde4fa5549","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_legacy_proxies.","id":"frs_legacy_proxies","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_legacy_proxies@1","mass":"conserve","outputs":[{"column":"legacy_jobseeker_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_health_condition_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"esa_support_group_proxy","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_legacy_proxies","stage_contract_sha256":"1ecf761cf3fa7180da15659e138e67a8654e58272b4fde28344aa27a874ad0aa","time_period":"2024"},"population":"age_tail.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"age_tail.boundary","citation":"","description":"Ownership boundary before frs_education_grant_split rewrites.","id":"frs_education_grant_split.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_education_grant_split.","id":"frs_education_grant_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_education_grant_split@1","mass":"conserve","outputs":[{"column":"disabled_students_allowance_eligible_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"education_grants","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_education_grant_split","stage_contract_sha256":"8718f014b90498c2cc7c754289775ca4a41452d9b978d8cbc5f34b64cbdc7df2","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_hmrc_spine_leaves.","id":"frs_hmrc_spine_leaves","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","education_grants","employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_hmrc_spine_leaves@1","mass":"conserve","outputs":[{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"ossben_identifiable_subset","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"srp_regular_code5","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_hmrc_spine_leaves","stage_contract_sha256":"fc65ca6f85857783db48ef2eda41e229d539d4420302e67c02df6e9a90ac3282","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_annual_income_spells.","id":"frs_annual_income_spells","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","employer_pension_contributions","employment_income","gender","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_annual_income_spells@1","mass":"conserve","outputs":[{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_annual_income_spells","stage_contract_sha256":"1dbdd0c9a6602be53b5527435d093a934218936d550da8b720973fc745c997ca","time_period":"2024"},"population":"frs_education_grant_split.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_education_grant_split.boundary","citation":"","description":"Run structural UK stage geographic_expand.","id":"geographic_expand","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.geographic_expand@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["person","person_clone_index","int64"],["benunit","benunit_clone_index","int64"],["household","source_year","int64"],["household","source_household_key","string"],["household","source_household_id","int64"],["household","household_clone_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"design","geographic_clones":1,"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"geographic_expand","stage_contract_sha256":"83140dea1a58e241d9df40b54592ea8c21578d1c279aff2e467b6fbf5dc482e5","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by geographic_expand.","id":"geographic_expand.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"source_year","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"source_household_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_clone_index","benunit.benunit_clone_index","household.source_year","household.source_household_key","household.source_household_id","household.household_clone_index"]},"population":"geographic_expand","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage geographic_identity.","id":"geographic_identity","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_clone_index","region","source_household_id","source_year"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.geographic_identity@1","mass":"conserve","outputs":[{"column":"geography_household_key","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"geographic_identity","stage_contract_sha256":"529efae366c4b0f7a1171b3c11dd8e56e91301c8b6389dc9bb3a9f7a7259efd1","time_period":"2024"},"population":"geographic_expand","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_take_up.","id":"frs_take_up","inputs":[{"columns":["frs_benunit_capital","is_married"],"entity":"benunit","rows":"all"},{"columns":["geography_household_key","household_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_benefit_reported","is_uc_claimant","months_since_last_birthday","pension_credit_reported","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"would_claim_uc_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_take_up","stage_contract_sha256":"78aec80408c50455b37308e5a977fb70df3ecf915d6c1e8f80715ad9b58e62cf","time_period":"2024"},"population":"geographic_expand","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_person_draws.","id":"frs_person_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_status","max_free_entitlement_hours_used"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_person_draws@1","mass":"conserve","outputs":[{"column":"would_claim_marriage_allowance","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"would_claim_scp","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"attends_private_school_random_draw","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"tax_free_childcare_spend_routed_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"is_apprentice","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_person_draws","stage_contract_sha256":"112c502d702c640dedbc2dca7d1cfe5ae43e0f0e3ba6acfe49c738dbbfb7e693","time_period":"2024"},"population":"geographic_expand","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_household_draws.","id":"frs_household_draws","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_household_draws@1","mass":"conserve","outputs":[{"column":"household_owns_tv","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"would_evade_tv_licence_fee","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residential_property_purchased_is_first_home","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"property_purchased","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_household_draws","stage_contract_sha256":"c74c63b09264319c4bf0049dabba00ecd0ce660beb7b53d6dae71518434bc949","time_period":"2024"},"population":"geographic_expand","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage frs_brma.","id":"frs_brma","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.frs_brma@1","mass":"conserve","outputs":[{"column":"brma","dtype":"string","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"frs_brma","stage_contract_sha256":"4285376b37299d375ed583b7f6b1747bd8b7cb1c8c0d19a36481156b5314a854","time_period":"2024"},"population":"geographic_expand","sources":["frs"],"structural":"none","weights":null},{"base":"geographic_expand","citation":"","description":"Freeze the assembled-spine gate population.","id":"frs_brma.checkpoint","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"frs_brma.checkpoint","citation":"","description":"Run structural UK stage spi_support_channel.","id":"spi_support_channel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_support_channel@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"spi_support_channel","stage_contract_sha256":"fbb39a593372e3f043232c22e583db56b0e341b6cfa4d09fcc4ccd41674d8d6d","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_support_channel.","id":"spi_support_channel.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["person.person_source_id","person.person_support_channel","person.person_support_clone_index","benunit.benunit_source_id","benunit.benunit_support_channel","benunit.benunit_support_clone_index","household.household_source_id","household.household_support_channel","household.household_support_clone_index","household.household_is_spi_synthetic"]},"population":"spi_support_channel","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"spi_support_channel","citation":"","description":"Run structural UK stage spi_income_band_donors.","id":"spi_income_band_donors","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.spi_income_band_donors@1","mass":"declared","outputs":[],"params":{"expand_cells":[["person","person_source_id","int64"],["person","person_support_channel","string"],["person","person_support_clone_index","int64"],["benunit","benunit_source_id","int64"],["benunit","benunit_support_channel","string"],["benunit","benunit_support_clone_index","int64"],["household","household_source_id","int64"],["household","household_support_channel","string"],["household","household_support_clone_index","int64"],["household","household_is_spi_synthetic","bool"],["household","household_is_spi_income_band_donor","bool"],["household","spi_income_band_donor_lower_bound","float64"],["person","person_is_spi_income_band_carrier","bool"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"spi_income_band_donors","stage_contract_sha256":"1895a652f6a0d3a7959b438d7545741b66606d4213df0d5a957d19ed52a9e371","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by spi_income_band_donors.","id":"spi_income_band_donors.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"person_source_id","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_channel","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_support_clone_index","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_source_id","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_channel","dtype":"string","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"benunit_support_clone_index","dtype":"int64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_source_id","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_channel","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_support_clone_index","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_synthetic","dtype":"bool","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"household_is_spi_income_band_donor","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"spi_income_band_donor_lower_bound","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"person_is_spi_income_band_carrier","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_spi_income_band_donor","household.spi_income_band_donor_lower_bound","person.person_is_spi_income_band_carrier"]},"population":"spi_income_band_donors","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_spi_income_spine.","id":"hmrc_spi_income_spine","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","rent_paid_as_boarder","rent_paid_as_lodger","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","maintenance_expenses","childcare_expenses","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","ssmg_reported","incapacity_benefit_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","ossben_identifiable_subset","srp_regular_code5","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_spi_income_spine@1","mass":"conserve","outputs":[{"column":"charitable_investment_gifts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"gift_aid","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_investment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employment_expenses","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_social_security_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_taxable_termination_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_miscellaneous_employment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_other_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_state_pension_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_employed_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_earned_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_total_investment_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"hmrc_spi_assessable_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"savings_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dividend_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employer_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"personal_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"individual_savings_account_interest_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"attendance_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"state_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"carers_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"winter_fuel_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"trading_loss","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"loss_relief","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ni_class_4_losses_brought_forward","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_income_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_unemployment_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_incapacity_benefit_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"self_employment_income_other_spells","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"private_pension_income_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"contributory_benefits_annual_share","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"annuity_and_trust_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"aa_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_sc_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"dla_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_m_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pip_dl_category","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_startup_period","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_includes_support_component","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"reports_rent_from_other_property","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"stands_as_sole_owner_or_partner_of_company","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_engaged_in_owned_company_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_is_main_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_trade","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_company_carries_on_property_business","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"person_is_spi_income_recipient","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"hmrc_spi_income_spine","stage_contract_sha256":"d4e912bc93bbd0345a608a47322937822f939a1d9ec95fb57ef2c293a2ed74a2","time_period":"2024"},"population":"spi_income_band_donors","sources":["frs"],"structural":"none","weights":null},{"base":"spi_income_band_donors","citation":"","description":"Ownership boundary before company_cars rewrites.","id":"company_cars.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage company_cars.","id":"company_cars","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient"],"entity":"person","rows":"all"}],"kernel":"uk.stage.company_cars@1","mass":"conserve","outputs":[{"column":"company_car_list_price","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_electric_range","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_capital_contributions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"company_car_private_fuel_provided","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"uc_is_in_gainful_self_employment","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_pay","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"hmrc_spi_employment_benefits","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"company_cars","stage_contract_sha256":"9c26f7fb7fbdf1b9f30930a732230c74967e320b60e6ba38462cf1fd04645353","time_period":"2024"},"population":"company_cars.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"company_cars.boundary","citation":"","description":"Ownership boundary before spi_employment_shell rewrites.","id":"spi_employment_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_employment_shell.","id":"spi_employment_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_clone_index","household_support_channel","household_support_clone_index","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","employment_status","gender","hmrc_spi_employment_benefits","hmrc_spi_pay","person_is_spi_income_recipient","private_pension_income","savings_interest_income","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_employment_shell@1","mass":"conserve","outputs":[{"column":"soc2020_major_group","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sic_industry_division","dtype":"int64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_sector","dtype":"string","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"spi_employment_shell","stage_contract_sha256":"4612c78bf72a36c34a6be652d15d1348fe3d99c873fa8d5821978816b59d1618","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage aps_occupation.","id":"aps_occupation","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_support_channel","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","employment_income","employment_sector","employment_status","gender","hmrc_spi_pay","person_is_spi_income_recipient","person_source_id","sic_industry_division","soc2020_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.aps_occupation@1","mass":"conserve","outputs":[{"column":"soc2020_sub_major_group","dtype":"string","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"aps_occupation","stage_contract_sha256":"644102f010870dccd28f8b980cbc5f648b7ff8ed140a0a5d9f23f02ada7f3d6d","time_period":"2024"},"population":"spi_employment_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_employment_shell.boundary","citation":"","description":"Ownership boundary before spi_housing_shell rewrites.","id":"spi_housing_shell.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_housing_shell.","id":"spi_housing_shell","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["council_tax_single_adult_raw","household_clone_index","household_support_channel","household_support_clone_index","ons_household_type","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","is_household_head","other_investment_income","private_pension_income","property_income","savings_interest_income","self_employment_income","soc2020_sub_major_group","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_housing_shell@1","mass":"conserve","outputs":[{"column":"tenure_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"accommodation_type","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_band","dtype":"string","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"num_bedrooms","dtype":"int64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_reported","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"rent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_interest_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"mortgage_capital_repayment","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"structural_insurance_payments","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_service_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"water_and_sewerage_charges","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"domestic_rates","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"subrent","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_rebate","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"housing_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"council_tax_benefit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"spi_housing_shell","stage_contract_sha256":"9a76a02ceddb2cd85819f57cd33feb44d645800ce963d0b18bad0f6de808fd81","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_wealth.","id":"was_wealth","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_wealth@1","mass":"conserve","outputs":[{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"corporate_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"private_pension_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gross_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"net_financial_wealth","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"savings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"num_vehicles","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cash_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"mortgage_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"consumer_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"directly_held_shares","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"unit_and_investment_trusts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_secured_debt","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"was_wealth","stage_contract_sha256":"015e9581671b86f1e0182785079fb29409431b6f5486458d684e6a8649cb8f4f","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage was_lisa.","id":"was_lisa","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.was_lisa@1","mass":"conserve","outputs":[{"column":"has_lifetime_isa","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_lifetime_isa_balance","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"was_lisa","stage_contract_sha256":"29e694a30f1551cdba3ae600113de1549b8f95fff4d695e663651a019f8304d6","time_period":"2024"},"population":"spi_housing_shell.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_housing_shell.boundary","citation":"","description":"Ownership boundary before regional_property_uprating rewrites.","id":"regional_property_uprating.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_property_uprating.","id":"regional_property_uprating","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_lifetime_isa_balance","household_support_channel","region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_property_uprating@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"owned_land","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"other_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"non_residential_property_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"regional_property_uprating","stage_contract_sha256":"f45db95fde3f93731c0925b9bf8c12021921df2e4908770acdec1e3c97889277","time_period":"2024"},"population":"regional_property_uprating.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"regional_property_uprating.boundary","citation":"","description":"Run structural UK stage high_value_home_support.","id":"high_value_home_support","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.high_value_home_support@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_high_value_support_copy","bool"],["household","high_value_support_copies","int64"],["household","high_value_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"high_value_home_support","stage_contract_sha256":"c8abc321c0407ba371ab4438359b79609aed0dba57dda69e572e83a76fce9723","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by high_value_home_support.","id":"high_value_home_support.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_high_value_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"high_value_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_high_value_support_copy","household.high_value_support_copies","household.high_value_support_copy_index"]},"population":"high_value_home_support","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage regional_home_values.","id":"regional_home_values","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_is_high_value_support_copy","region","source_household_id","tenure_type"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.regional_home_values@1","mass":"conserve","outputs":[{"column":"main_residence_value","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_wealth","dtype":"float64","entity":"household","ownership":"produced","rewrite":true,"rows":"all"},{"column":"main_residence_value_rank_basis","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"regional_home_values","stage_contract_sha256":"48e8a622c8830a54a4c48d9d8cb4f011751aa05d7cb5f6c7f0e06e1121b720b1","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_components.","id":"property_components","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_clone_index","main_residence_value_rank_basis","non_residential_property_value","other_residential_property_value","region","subrent"],"entity":"household","rows":"all"},{"columns":["age","is_household_head","person_is_spi_income_recipient","person_support_channel"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_components@1","mass":"conserve","outputs":[{"column":"property_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_finance_costs","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"property_rental_income","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"property_components","stage_contract_sha256":"1e513a7c6644d48ee0952a6d3f3151df3c6a8dd2289bec0b355f40fbbd10475d","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage property_purchases.","id":"property_purchases","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["non_residential_property_value","other_residential_property_value","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.property_purchases@1","mass":"conserve","outputs":[{"column":"additional_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"non_residential_property_purchased","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"property_purchases","stage_contract_sha256":"9e4913cd5c7e23feb955e0640e9ea8100fc3a9da9798856d58931ea61da60526","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage nts_bus_travel.","id":"nts_bus_travel","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.nts_bus_travel@1","mass":"conserve","outputs":[{"column":"local_bus_use_band","dtype":"int64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_in_london_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"other_local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_trips","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"bus_pass_eligible","dtype":"bool","entity":"person","ownership":"produced","rows":"all"},{"column":"local_bus_single_fare_share","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"household_local_bus_trips","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"nts_bus_travel","stage_contract_sha256":"6f6d0b6b7d0d5450854e9133382027ed3c83b6e26298eb97d68a478cf05b1f5c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage lcfs_consumption.","id":"lcfs_consumption","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share"],"entity":"person","rows":"all"}],"kernel":"uk.stage.lcfs_consumption@1","mass":"conserve","outputs":[{"column":"food_and_non_alcoholic_beverages_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"alcohol_and_tobacco_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"clothing_and_footwear_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"housing_water_and_electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"household_furnishings_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"health_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"transport_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"communication_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"recreation_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"education_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"restaurants_and_hotels_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"miscellaneous_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"petrol_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"diesel_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_fare_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"domestic_energy_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"electricity_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"gas_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"liquid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"solid_fuel_consumption","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"has_fuel_consumption","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"uses_energy_prepayment_meter","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"wine_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"spirits_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_fermented_litres","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"beer_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cider_draught_share","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigarettes_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"hand_rolling_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"cigars_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"other_tobacco_grams_per_week","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_gaming_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"pool_betting_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bingo_net_receipts","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"national_lottery_ticket_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_casino_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_lower_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_standard_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"machine_games_higher_rate_net_takings","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"land_based_fixed_odds_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_uk_horseracing_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"remote_other_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"betting_exchange_commission","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"horse_and_dog_pool_betting_net_losses","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"car_fuel_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_co2_emissions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_engine_size","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"car_first_registration_date","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"car_is_ved_exempt","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"lcfs_consumption","stage_contract_sha256":"c15ed1f177d8d4ec3446a379352b437bccd7392f711cf028a5bc6f4ecabb10b4","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_vat.","id":"etb_vat","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_vat@1","mass":"conserve","outputs":[{"column":"full_rate_vat_expenditure_rate","dtype":"float64","entity":"household","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"etb_vat","stage_contract_sha256":"216b28835eba7a952243134dffbb45a1b3a0cdff2e3e7d0e94e2acbf754052b5","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage etb_services.","id":"etb_services","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt"],"entity":"person","rows":"all"}],"kernel":"uk.stage.etb_services@1","mass":"conserve","outputs":[{"column":"dfe_education_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"bus_subsidy_spending","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"rail_usage","dtype":"float64","entity":"household","ownership":"produced","rows":"all"},{"column":"a_and_e_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"admitted_patient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"outpatient_visits","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_a_and_e_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_admitted_patient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"nhs_outpatient_spending","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"etb_services","stage_contract_sha256":"9971bdb17ce943cdf00be9d2a5d6d9dec1bfdc652e3d392464d4defb9531424c","time_period":"2024"},"population":"high_value_home_support","sources":["frs"],"structural":"none","weights":null},{"base":"high_value_home_support","citation":"","description":"Ownership boundary before uc_reporter_redraw rewrites.","id":"uc_reporter_redraw.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_reporter_redraw.","id":"uc_reporter_redraw","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_reporter_redraw@1","mass":"conserve","outputs":[{"column":"universal_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"uc_reporter_redraw","stage_contract_sha256":"1b8f99bd8bc50e3e5dacbb63b75e6b8da0c42c1c8c508c492d078e08ddab0e66","time_period":"2024"},"population":"uc_reporter_redraw.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_reporter_redraw.boundary","citation":"","description":"Ownership boundary before spi_benefit_coherence rewrites.","id":"spi_benefit_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage spi_benefit_coherence.","id":"spi_benefit_coherence","inputs":[{"columns":["benunit_support_channel","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["aa_category","age","attendance_allowance_reported","dla_m_category","dla_m_reported","dla_sc_category","dla_sc_reported","employment_income","hours_worked","incapacity_benefit_reported","is_uc_claimant","person_source_id","person_support_channel","pip_dl_category","pip_dl_reported","pip_m_category","pip_m_reported","private_pension_income","self_employment_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.spi_benefit_coherence@1","mass":"conserve","outputs":[{"column":"jsa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_income_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"income_support_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"working_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_tax_credit_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"sda_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"ssmg_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"iidb_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"afcs_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"bsp_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"war_widows_pension_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"widowed_parents_allowance_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"jsa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"esa_contrib_reported","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_enhanced_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"is_severely_disabled_for_benefits","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"receives_benefits_in_own_right","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"spi_benefit_coherence","stage_contract_sha256":"2c0162b22d5fcab2296f654177c9f87710184c20fb323cff25710ee0667fcdef","time_period":"2024"},"population":"spi_benefit_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"spi_benefit_coherence.boundary","citation":"","description":"Ownership boundary before uc_capital_coherence rewrites.","id":"uc_capital_coherence.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_capital_coherence.","id":"uc_capital_coherence","inputs":[{"columns":["benunit_support_channel","dependent_children","is_married"],"entity":"benunit","rows":"all"},{"columns":["high_value_support_copy_index","household_clone_index","household_is_high_value_support_copy","household_support_channel","household_support_clone_index","non_residential_property_secured_debt","non_residential_property_value","other_residential_property_secured_debt","other_residential_property_value","owned_land","region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","individual_savings_account_interest_income","is_benunit_head","is_household_head","is_parent","other_investment_income","person_support_channel","savings_interest_income","universal_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_capital_coherence@1","mass":"conserve","outputs":[{"column":"uc_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"pension_credit_reported_capital","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"frs_benunit_capital","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"uc_capital_coherence","stage_contract_sha256":"67478de55540a138448d9da387a64e5e2afbb17caf17b244bc3baaa34de56022","time_period":"2024"},"population":"uc_capital_coherence.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_capital_coherence.boundary","citation":"","description":"Ownership boundary before uc_take_up rewrites.","id":"uc_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_take_up.","id":"uc_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"uc_take_up","stage_contract_sha256":"8bb7a8a3833971a8ca57f8af3b8ef37061972fc5995c4224d8374e0b4bf6aeb4","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_legacy_closure_take_up.","id":"uc_legacy_closure_take_up","inputs":[{"columns":["frs_benunit_capital","would_claim_uc"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","child_tax_credit_reported","esa_income_reported","housing_benefit_reported","income_support_reported","is_claimant_or_partner","jsa_income_reported","universal_credit_reported","working_tax_credit_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_legacy_closure_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_uc_at_legacy_closure","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"uc_legacy_closure_take_up","stage_contract_sha256":"7eadda2888a20a7089da46c9b41ac7fdd4a73a4f2c27ae4be716f8af2ee1836f","time_period":"2024"},"population":"uc_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"uc_take_up.boundary","citation":"","description":"Ownership boundary before pension_credit_take_up rewrites.","id":"pension_credit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage pension_credit_take_up.","id":"pension_credit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.pension_credit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_pc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"has_mixed_age_couple_pension_credit_saving","dtype":"bool","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"pension_credit_take_up","stage_contract_sha256":"e887dbbb186cb79f6d3fc9a629269dbac4a43381b9574df65353b816f01a3a35","time_period":"2024"},"population":"pension_credit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"pension_credit_take_up.boundary","citation":"","description":"Ownership boundary before child_benefit_take_up rewrites.","id":"child_benefit_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage child_benefit_take_up.","id":"child_benefit_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.child_benefit_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_child_benefit","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"child_benefit_opts_out","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"child_benefit_take_up","stage_contract_sha256":"25638e88bc9b9236b4ec6ec8ee9a05306f0aa960fcc165330a2e25eea65124ab","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage uc_deduction_attributes.","id":"uc_deduction_attributes","inputs":[{"columns":["child_benefit_opts_out","frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age"],"entity":"person","rows":"all"}],"kernel":"uk.stage.uc_deduction_attributes@1","mass":"conserve","outputs":[{"column":"uc_deduction_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_type_random_draw","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_latent_deduction_rate","dtype":"float64","entity":"benunit","ownership":"produced","rows":"all"},{"column":"uc_deduction_combination","dtype":"string","entity":"benunit","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"uc_deduction_attributes","stage_contract_sha256":"6bc7e3ec1a5712e23ca1b7d2956e8b17f2526b94431ba3e5a9294b04658e4fe0","time_period":"2024"},"population":"child_benefit_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"child_benefit_take_up.boundary","citation":"","description":"Run structural UK stage cgt_support_split.","id":"cgt_support_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_support_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_support_copy","bool"],["household","cgt_support_copies","int64"],["household","cgt_support_copy_index","int64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"cgt_support_split","stage_contract_sha256":"7667fce8bb5b4bc40a2bc41fa55b03528db29e5715a84b0a55a3f08134e02f2d","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_support_split.","id":"cgt_support_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_support_copy","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copies","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_support_copy_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_support_copy","household.cgt_support_copies","household.cgt_support_copy_index"]},"population":"cgt_support_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_support_split","citation":"","description":"Run structural UK stage cgt_incidence_clone.","id":"cgt_incidence_clone","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_clone@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_capital_gains_clone","bool"],["person","capital_gains","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"cgt_incidence_clone","stage_contract_sha256":"a19b460e2bb32c6e40d302b7f25e2e22b082d61f8e39a338c58e5ee87c0cfa1c","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_incidence_clone.","id":"cgt_incidence_clone.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_capital_gains_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_capital_gains_clone","person.capital_gains"]},"population":"cgt_incidence_clone","sources":[],"structural":"none","weights":null},{"base":"cgt_incidence_clone","citation":"","description":"Ownership boundary before hmrc_cgt_gains_spine rewrites.","id":"hmrc_cgt_gains_spine.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_gains_spine.","id":"hmrc_cgt_gains_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cgt_support_copies","corporate_wealth","gross_financial_wealth","household_is_cgt_support_copy","non_residential_property_value","other_residential_property_value","region"],"entity":"household","rows":"all"},{"columns":["age","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_gains_spine@1","mass":"conserve","outputs":[{"column":"capital_gains","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"hmrc_cgt_gains_spine","stage_contract_sha256":"1dd10a33849c0377b6e79f5e5077ebad741b8f13de44896f05e7322700d90e3e","time_period":"2024"},"population":"hmrc_cgt_gains_spine.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"hmrc_cgt_gains_spine.boundary","citation":"","description":"Run structural UK stage cgt_incidence_anchor.","id":"cgt_incidence_anchor","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["household_is_capital_gains_clone","region"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","dividend_income","employment_income","individual_savings_account_interest_income","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_incidence_anchor@1","mass":"conserve","outputs":[],"params":{"expand_cells":[],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"cgt_incidence_anchor","stage_contract_sha256":"32fc32cf1210f1236b7bbe2c377f0266c1989a2e9431c5875e4182a157b6e5fe","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":"cgt_incidence_anchor","citation":"","description":"Run structural UK stage cgt_residential_split.","id":"cgt_residential_split","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains"],"entity":"person","rows":"all"}],"kernel":"uk.stage.expand.cgt_residential_split@1","mass":"conserve","outputs":[],"params":{"expand_cells":[["household","household_is_cgt_residential_clone","bool"],["household","cgt_residential_clone_index","int64"],["person","cgt_residential_probability","float64"],["person","capital_gains_residential_property","float64"]],"expand_weight_entity":"household","expand_weight_kind":"importance","numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"cgt_residential_split","stage_contract_sha256":"77a8a95621aeb8fcfa1dc2255e5b4a73427528f4838e3df60ed20fba7bb1640a","time_period":"2024"},"population":null,"sources":["frs"],"structural":"expand","weights":null},{"base":null,"citation":"","description":"Own the cells materialized by cgt_residential_split.","id":"cgt_residential_split.owned","inputs":[],"kernel":"uk.claim@1","mass":"conserve","outputs":[{"column":"household_is_cgt_residential_clone","dtype":"bool","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_clone_index","dtype":"int64","entity":"household","ownership":"produced","rows":"all"},{"column":"cgt_residential_probability","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_residential_property","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"materialized_expand_outputs":["household.household_is_cgt_residential_clone","household.cgt_residential_clone_index","person.cgt_residential_probability","person.capital_gains_residential_property"]},"population":"cgt_residential_split","sources":[],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage hmrc_cgt_asset_type_spine.","id":"hmrc_cgt_asset_type_spine","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["corporate_wealth","gross_financial_wealth","other_residential_property_value","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","capital_gains","capital_gains_residential_property","dividend_income","property_income","reports_rent_from_other_property","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.hmrc_cgt_asset_type_spine@1","mass":"conserve","outputs":[{"column":"capital_gains_asset_type","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"capital_gains_badr","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"hmrc_cgt_asset_type_spine","stage_contract_sha256":"a0a0d7d311f217d9c8149656efd4b7baf7ee11c2eca2a904de694253ccd171e7","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage salary_sacrifice.","id":"salary_sacrifice","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr"],"entity":"person","rows":"all"}],"kernel":"uk.stage.salary_sacrifice@1","mass":"conserve","outputs":[{"column":"pension_contributions_via_salary_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"pension_contributions_via_bonus_sacrifice","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"employee_pension_contributions","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"employment_income","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"},{"column":"salary_sacrifice_pre_conversion_pay","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"salary_sacrifice","stage_contract_sha256":"4df6174d0784667dc5aab7012214886908d7d2b1cba917da67bf30b9a3334281","time_period":"2024"},"population":"cgt_residential_split","sources":["frs"],"structural":"none","weights":null},{"base":"cgt_residential_split","citation":"","description":"Ownership boundary before carers_allowance_take_up rewrites.","id":"carers_allowance_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage carers_allowance_take_up.","id":"carers_allowance_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.stage.carers_allowance_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_carers_allowance","dtype":"bool","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"carers_allowance_take_up","stage_contract_sha256":"7cf2cf79bd0f9a92f2a334081a35c8bfe24a0b841131fb9a8b8aceee6caf2dba","time_period":"2024"},"population":"carers_allowance_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"carers_allowance_take_up.boundary","citation":"","description":"Ownership boundary before student_loans rewrites.","id":"student_loans.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage student_loans.","id":"student_loans","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region"],"entity":"household","rows":"all"},{"columns":["age","age_left_full_time_education","current_education","highest_education","reports_outstanding_student_loan","student_loan_repayments","student_loans","would_claim_carers_allowance"],"entity":"person","rows":"all"}],"kernel":"uk.stage.student_loans@1","mass":"conserve","outputs":[{"column":"student_loan_plan","dtype":"string","entity":"person","ownership":"produced","rows":"all"},{"column":"student_loan_balance","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"student_loans","stage_contract_sha256":"507ca0160144d8f5ea1b2483b812658ee8b49854e2388d50a8c59ca58f2ca9ee","time_period":"2024"},"population":"student_loans.boundary","sources":["frs"],"structural":"none","weights":null},{"base":"student_loans.boundary","citation":"","description":"Ownership boundary before childcare_copy_take_up rewrites.","id":"childcare_copy_take_up.boundary","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.identity@1","mass":"conserve","outputs":[],"params":{},"population":null,"sources":[],"structural":"filter","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage childcare_copy_take_up.","id":"childcare_copy_take_up","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_uc_childcare","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.childcare_copy_take_up@1","mass":"conserve","outputs":[{"column":"would_claim_tfc","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_extended_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_universal_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"would_claim_targeted_childcare","dtype":"bool","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"maximum_extended_childcare_hours_usage","dtype":"float64","entity":"benunit","ownership":"produced","rewrite":true,"rows":"all"},{"column":"max_free_entitlement_hours_used","dtype":"float64","entity":"person","ownership":"produced","rewrite":true,"rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"childcare_copy_take_up","stage_contract_sha256":"cdbb7e135b16fac7ca1bcd6c54222821e2976b4da99f10a60913efd0ba03c13c","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage private_school_attendance.","id":"private_school_attendance","inputs":[{"columns":["frs_benunit_capital","is_married","dependent_children","liable_for_share_of_household_rent","benunit_clone_index","would_claim_child_benefit","child_benefit_opts_out","would_claim_pc","would_claim_uc","would_claim_tfc","would_claim_extended_childcare","would_claim_universal_childcare","would_claim_targeted_childcare","would_claim_uc_childcare","maximum_extended_childcare_hours_usage","benunit_source_id","benunit_support_channel","benunit_support_clone_index","uc_reported_capital","pension_credit_reported_capital","would_claim_uc_at_legacy_closure","has_mixed_age_couple_pension_credit_saving","uc_deduction_random_draw","uc_deduction_type_random_draw","uc_latent_deduction_rate","uc_deduction_combination"],"entity":"benunit","rows":"all"},{"columns":["region","imd_decile","tenure_type","accommodation_type","num_bedrooms","council_tax_reported","council_tax_band","council_tax_band_observed","council_tax_rebate","council_tax_single_adult_raw","water_and_sewerage_charges","domestic_rates","rent","subrent","mortgage_interest_repayment","mortgage_capital_repayment","structural_insurance_payments","housing_service_charges","external_child_payments","ons_household_type","council_tax","source_year","source_household_key","source_household_id","household_clone_index","geography_household_key","household_owns_tv","would_evade_tv_licence_fee","main_residential_property_purchased_is_first_home","property_purchased","brma","household_source_id","household_support_channel","household_support_clone_index","household_is_spi_synthetic","household_is_spi_income_band_donor","spi_income_band_donor_lower_bound","owned_land","property_wealth","corporate_wealth","private_pension_wealth","gross_financial_wealth","net_financial_wealth","main_residence_value","other_residential_property_value","non_residential_property_value","savings","num_vehicles","cash_isa","stocks_and_shares_isa","mortgage_debt","consumer_debt","directly_held_shares","unit_and_investment_trusts","other_residential_property_secured_debt","non_residential_property_secured_debt","household_lifetime_isa_balance","household_is_high_value_support_copy","high_value_support_copies","high_value_support_copy_index","main_residence_value_rank_basis","additional_residential_property_purchased","non_residential_property_purchased","household_local_bus_trips","food_and_non_alcoholic_beverages_consumption","alcohol_and_tobacco_consumption","clothing_and_footwear_consumption","housing_water_and_electricity_consumption","household_furnishings_consumption","health_consumption","transport_consumption","communication_consumption","recreation_consumption","education_consumption","restaurants_and_hotels_consumption","miscellaneous_consumption","petrol_spending","diesel_spending","bus_fare_spending","domestic_energy_consumption","electricity_consumption","gas_consumption","liquid_fuel_consumption","solid_fuel_consumption","has_fuel_consumption","uses_energy_prepayment_meter","beer_litres","cider_litres","wine_litres","spirits_litres","other_fermented_litres","beer_draught_share","cider_draught_share","cigarettes_per_week","hand_rolling_tobacco_grams_per_week","cigars_grams_per_week","other_tobacco_grams_per_week","remote_gaming_net_losses","pool_betting_net_receipts","bingo_net_receipts","national_lottery_ticket_spending","land_based_casino_net_losses","machine_games_lower_rate_net_takings","machine_games_standard_rate_net_takings","machine_games_higher_rate_net_takings","land_based_fixed_odds_betting_net_losses","remote_uk_horseracing_betting_net_losses","remote_other_betting_net_losses","betting_exchange_commission","horse_and_dog_pool_betting_net_losses","full_rate_vat_expenditure_rate","dfe_education_spending","rail_subsidy_spending","bus_subsidy_spending","rail_usage","household_is_cgt_support_copy","cgt_support_copies","cgt_support_copy_index","household_is_capital_gains_clone","household_is_cgt_residential_clone","cgt_residential_clone_index"],"entity":"household","rows":"all"},{"columns":["age","gender","months_since_last_birthday","marital_status","hours_worked","care_hours","frs_care_hours_in_household","is_household_head","is_benunit_head","is_parent","is_blind","is_uc_claimant","is_claimant_or_partner","is_hbai_dependent_child","uc_is_in_startup_period","uc_is_in_gainful_self_employment","rent_paid_as_boarder","rent_paid_as_lodger","employment_income","self_employment_income","trading_loss","loss_relief","ni_class_4_losses_brought_forward","private_pension_income","annuity_and_trust_income","individual_savings_account_interest_income","savings_interest_income","dividend_income","other_investment_income","stands_as_sole_owner_or_partner_of_company","is_engaged_in_owned_company_trade","owned_company_is_main_employment","owned_company_carries_on_trade","owned_company_carries_on_property_business","owned_company_income_share","property_income","reports_rent_from_other_property","maintenance_income","spousal_maintenance_income","miscellaneous_income","private_transfer_income","lump_sum_income","student_loan_repayments","statutory_sick_pay","statutory_maternity_pay","statutory_paternity_pay","student_loans","access_fund","education_grants","healthy_start_vouchers","free_school_breakfasts","free_school_fruit_veg","free_school_meals","council_tax_benefit_reported","maintenance_expenses","childcare_expenses","personal_pension_contributions","employee_pension_contributions","pension_contributions_via_salary_sacrifice","salary_sacrifice_reported","salary_sacrifice_asked","frs_bonus_amount","frs_company_car_price_band","frs_company_car_fuel_code","frs_company_car_private_fuel","frs_company_car_contribution","child_benefit_reported","income_support_reported","housing_benefit_reported","attendance_allowance_reported","dla_sc_reported","dla_m_reported","iidb_reported","carers_allowance_reported","would_claim_carers_allowance","sda_reported","afcs_reported","ssmg_reported","pension_credit_reported","child_tax_credit_reported","working_tax_credit_reported","state_pension_reported","winter_fuel_allowance_reported","incapacity_benefit_reported","universal_credit_reported","pip_m_reported","pip_dl_reported","jsa_contrib_reported","jsa_income_reported","esa_contrib_reported","esa_income_reported","esa_includes_support_component","bsp_reported","war_widows_pension_reported","widowed_parents_allowance_reported","relationship_to_head","ons_family_role","ons_family_index","is_looked_after_by_local_authority","child_limit_non_parental_caring_exception","employment_status","employment_sector","sic_industry_division","soc2020_major_group","aa_category","dla_sc_category","dla_m_category","pip_m_category","pip_dl_category","is_disabled_for_benefits","is_enhanced_disabled_for_benefits","is_severely_disabled_for_benefits","current_education","highest_education","is_in_non_advanced_education","is_in_approved_training","age_started_or_accepted_current_education_or_training","is_before_universal_credit_qualifying_young_person_terminal_date","adult_ema","child_ema","receives_benefits_in_own_right","reports_outstanding_student_loan","age_left_full_time_education","legacy_jobseeker_proxy","esa_health_condition_proxy","esa_support_group_proxy","disabled_students_allowance_eligible_expenses","hmrc_spi_pay","hmrc_spi_unemployment_benefit_income","hmrc_spi_incapacity_benefit_income","ossben_identifiable_subset","srp_regular_code5","employer_pension_contributions","employment_income_annual_share","employment_income_other_spells","self_employment_income_annual_share","self_employment_income_other_spells","private_pension_income_annual_share","contributory_benefits_annual_share","person_clone_index","max_free_entitlement_hours_used","would_claim_marriage_allowance","would_claim_scp","attends_private_school_random_draw","tax_free_childcare_spend_routed_share","is_apprentice","person_source_id","person_support_channel","person_support_clone_index","person_is_spi_income_band_carrier","charitable_investment_gifts","gift_aid","hmrc_spi_employment_benefits","hmrc_spi_employment_expenses","hmrc_spi_other_social_security_income","hmrc_spi_taxable_termination_pay","hmrc_spi_miscellaneous_employment_income","hmrc_spi_other_income","hmrc_spi_state_pension_income","property_finance_costs","hmrc_spi_employed_income","hmrc_spi_total_earned_income","hmrc_spi_total_investment_income","hmrc_spi_assessable_income","person_is_spi_income_recipient","company_car_list_price","company_car_co2_emissions","company_car_fuel_type","company_car_electric_range","company_car_capital_contributions","company_car_private_fuel_provided","soc2020_sub_major_group","student_loan_balance","has_lifetime_isa","lifetime_isa_balance","property_rental_income","local_bus_use_band","bus_in_london_trips","other_local_bus_trips","local_bus_trips","bus_pass_eligible","local_bus_single_fare_share","car_fuel_type","car_co2_emissions","car_engine_size","car_first_registration_date","car_is_ved_exempt","a_and_e_visits","admitted_patient_visits","outpatient_visits","nhs_a_and_e_spending","nhs_admitted_patient_spending","nhs_outpatient_spending","capital_gains","cgt_residential_probability","capital_gains_residential_property","capital_gains_asset_type","capital_gains_badr","pension_contributions_via_bonus_sacrifice","salary_sacrifice_pre_conversion_pay","student_loan_plan"],"entity":"person","rows":"all"}],"kernel":"uk.stage.private_school_attendance@1","mass":"conserve","outputs":[{"column":"attends_private_school","dtype":"bool","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"private_school_attendance","stage_contract_sha256":"70e1df19044f2d4fcebd5db1984c1385151feea6a9d0642d9b11da045a4f83ee","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage isa_subscriptions.","id":"isa_subscriptions","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["cash_isa","region","stocks_and_shares_isa"],"entity":"household","rows":"all"},{"columns":["age","attends_private_school","dividend_income","employment_income","gender","has_lifetime_isa","individual_savings_account_interest_income","lifetime_isa_balance","loss_relief","miscellaneous_income","private_pension_income","property_income","savings_interest_income","self_employment_income","state_pension_reported"],"entity":"person","rows":"all"}],"kernel":"uk.stage.isa_subscriptions@1","mass":"conserve","outputs":[{"column":"cash_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_balance","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"cash_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"stocks_and_shares_isa_subscriptions","dtype":"float64","entity":"person","ownership":"produced","rows":"all"},{"column":"lifetime_isa_payments","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"isa_subscriptions","stage_contract_sha256":"95ea8da9ae254fa8bc7f0f36edee3247987468a53ebc516fda4d3606e93b894f","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null},{"artifact_outputs":[{"name":"stage_evidence","type":{"name":"microcosm.stage-evidence","schema_version":1}}],"base":null,"citation":"","description":"Run UK spine stage trading_allowance_receipts.","id":"trading_allowance_receipts","inputs":[{"columns":["frs_benunit_capital"],"entity":"benunit","rows":"all"},{"columns":["region","source_household_id"],"entity":"household","rows":"all"},{"columns":["age","lifetime_isa_payments","self_employment_income"],"entity":"person","rows":"all"}],"kernel":"uk.stage.trading_allowance_receipts@1","mass":"conserve","outputs":[{"column":"self_employment_gross_receipts","dtype":"float64","entity":"person","ownership":"produced","rows":"all"}],"params":{"numerical_dependencies":[["policyengine-uk","2.174.1"],["policyengine-core","3.33.3"],["numpy","2.4.6"],["pandas","3.0.3"],["scikit-learn","1.8.0"],["quantile-forest","1.4.2"],["microdf-python","1.3.0"]],"stage":"trading_allowance_receipts","stage_contract_sha256":"f91329806570bd874750845816de96b28160e766e8e26bf1174cb0809ad8ff43","time_period":"2024"},"population":"childcare_copy_take_up.boundary","sources":["frs"],"structural":"none","weights":null}],"sources":[{"codec":"csv-tables","description":"Content-bound UK FRS and donor fixture/source bundle.","name":"frs"}]} diff --git a/test_support/microcosm_build/uk_clone_first_frames.py b/test_support/microcosm_build/uk_clone_first_frames.py index fb6fe821a6..5ed172d0ab 100644 --- a/test_support/microcosm_build/uk_clone_first_frames.py +++ b/test_support/microcosm_build/uk_clone_first_frames.py @@ -1,13 +1,13 @@ """Synthetic UK id frames for the copy-id tests (microcosm#931). -The clone-first spine (#931 group (b)) will mint geographic clone ``k`` of -every FRS row at ``id + k x 10**9`` (the copy-id table's -``geographic_support`` field) before any copy layer runs. This module builds -such frames from synthetic sernums, with ids as ``frs_spine`` makes them -(``benunit_id`` sernum x 100 + b, ``person_id`` sernum x 1000 + p), and copies -households through the six copy layers in the spine's order: the SPI copy, -the income band donor, the high-value copies, the CGT support copies, the -incidence twin and the residential arms. Each layer picks its households by +The clone-first spine's ``geographic_expand`` stage (#931) mints geographic +clone ``k`` of every FRS row at ``id + k x 10**9`` (the copy-id table's +``geographic_support`` field) before any other copy layer runs. This module +builds such frames from synthetic sernums, with ids as ``frs_spine`` makes +them (``benunit_id`` sernum x 100 + b, ``person_id`` sernum x 1000 + p), and +copies households through the six copy layers in the spine's order: the SPI +copy, the income band donor, the high-value copies, the CGT support copies, +the incidence twin and the residential arms. Each layer picks its households by source id alone, so a K=1 frame's copies are exactly its clone 0's in a K=3 frame. ``rule="table"`` mints every copy through the copy-id table, as the stages do; ``rule="retired"`` replays the rule the table replaced (ten to the @@ -143,9 +143,15 @@ def clone_first_frame(frame: Frame, n_clones: int) -> Frame: field), ``household_clone_index`` ``k`` and ``source_household_id`` the clone-0 id (added when the frame has none), at the weight ``w / n_clones`` of its household (the same float in every clone), as the - clone-first spine will (#931 (b)). Every other column is copied - unchanged; the clones carry the frame's weight kind and time period, and - no mass record. + clone-first spine's ``geographic_expand`` stage does (#931). Every other + column is copied unchanged; the clones carry the frame's weight kind and + time period, and no mass record. + + It writes only the household clone index and source id, not the person + and benefit-unit clone indices, ``source_year`` or + ``source_household_key``, so ``spi_support_channel`` refuses its frames; + tests that run that stage clone through + ``geographic_expand.expand_uk_geographic_clones``. """ from microcosm.build.uk_runtime.national_frame import ( diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index 81061b69bd..9664603efc 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -953,6 +953,16 @@ def build_manifest( }, "effective_mass_coverage": EFFECTIVE_MASS_COVERAGE, "family_coverage": { + # The geographic expansion gives every household K clones that + # split its design weight equally (microcosm#931). Its one mass + # record keeps the household total to the split's rounding at any + # K, one clone included. The record's reason names no K, and the + # stage's transform refuses a declared reason other than it, so + # the terminal gate demands that receipt like any other stage's. + "geographic_expand": _source_stage_family_coverage_contract( + stage_name="geographic_expand", + candidate_source=candidate_source, + ), "cgt_incidence_clone": _source_stage_family_coverage_contract( stage_name="cgt_incidence_clone", candidate_source=candidate_source, diff --git a/tools/graph_uk_spine_fixture.py b/tools/graph_uk_spine_fixture.py index d45e3fc448..5ad2892599 100644 --- a/tools/graph_uk_spine_fixture.py +++ b/tools/graph_uk_spine_fixture.py @@ -1,12 +1,16 @@ #!/usr/bin/env python3 """Regenerate the hermetic charter-H2 UK spine parity fixture. -The oracle is the legacy :class:`microcosm.build.plan.StagePlan`: all 31 -stages are the current production transform classes. Private source files -are replaced only through their supported parsed-input seams. The bundle's -``fixture.json`` is deliberately data-only so the graph's unbound UK registry -can reconstruct the same transforms; no recorded stage deltas are written or -replayed. +The oracle is the legacy :class:`microcosm.build.plan.StagePlan` over every +stage of the packaged UK spine (``UK_FIXTURE_STAGE_COUNT``), each the current +production transform class. The stages up to ``frs_annual_income_spells`` run +on the survey households; ``geographic_expand`` then gives every household K +geographic clones (K is ``geographic_clones`` in the fixture's config, one +here), ``geographic_identity`` keys each clone, and every later stage runs on +the clones (microcosm#931). Private source files are replaced only through +their supported parsed-input seams. The bundle's ``fixture.json`` is +deliberately data-only so the graph's unbound UK registry can reconstruct the +same transforms; no recorded stage deltas are written or replayed. The oracle's identity is machine-specific, so it is never pinned: ``legacy_oracle_identity`` recomputes it in the acceptance test's own process. @@ -95,6 +99,12 @@ uk_frs_spine_seed_frame, ) from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform +from microcosm.build.uk_runtime.geographic_expand import ( + UKGeographicExpandStageTransform, +) +from microcosm.build.uk_runtime.geographic_identity import ( + UKGeographicIdentityStageTransform, +) from microcosm.build.uk_runtime.graph import uk_spine_graph from microcosm.build.uk_runtime.high_value_home_support import ( UKHighValueHomeSupportStageTransform, @@ -225,9 +235,14 @@ #: certified-pair exclusions); moves whenever a spine stage is added. UK_FIXTURE_STAGE_COUNT = 56 _QRF_ESTIMATORS = 4 +#: The fixture spine's geographic clone count K (microcosm#931). The config +#: records it, the oracle builds geographic_expand and its engine with it, as +#: the graph registry's fixture transforms do from the config, and the +#: committed graph carries it on geographic_expand's node. +_GEOGRAPHIC_CLONES = 1 # These are the complete object-string surface observed in the unchanged -# legacy 40-stage output. Graph storage uses pandas StringDtype/python. +# legacy output. Graph storage uses pandas StringDtype/python. _NORMALIZED_STRING_COLUMNS: Mapping[str, tuple[str, ...]] = { "person": ( "gender", @@ -411,11 +426,26 @@ def _frs_tables() -> dict[str, pd.DataFrame]: # region-stratified SPI prior allocation changes weighted person mass. has_child = household_id % 5 == 0 region = (8, 11, 12, 1)[household_id % 4] + # GVTREGNO 8 (London) and 1 (North East) are in England, 11 is Wales + # and 12 Scotland; no fixture household is in Northern Ireland (IMDN). + imd_variable = ("IMDE", "IMDW", "IMDS", "IMDE")[household_id % 4] households.append( { "SERNUM": household_id, "GROSS4": 1.0 + household_id / 100.0, "GVTREGNO": region, + # The address's IMD decile on its own nation's index + # (microcosm#931): deciles 1-10 in turn among each region's + # households, and blank, no decile, on every seventeenth + # household; the other nations' variables are blank. + **{ + variable: ( + 1 + (household_id // 4) % 10 + if variable == imd_variable and household_id % 17 + else "" + ) + for variable in ("IMDE", "IMDW", "IMDS", "IMDN") + }, "PTENTYP2": 5 + household_id % 2, "TYPEACC": 1 + household_id % 4, "BEDROOM6": 1 + household_id % 4, @@ -2304,11 +2334,14 @@ def _build_implementations( cgt_distribution: HMRCCapitalGainsJointDistribution, cgt_parameters: UKCGTPolicyParameters, cgt_badr_parameters: UKCGTBADRParameters, + geographic_clones: int, cgt_asset_type_facts: HMRCCGTAssetTypeFacts | None = None, property_rental_facts: HMRCPropertyRentalFacts | None = None, property_purchase_targets: UKPropertyPurchaseTargets | None = None, ) -> tuple[dict[str, object], dict[str, Frame]]: - engine = PolicyEngineUKEngine() + # The engine refuses a frame carrying another clone count than the + # spine's, as the graph registry's fixture engine does (microcosm#931). + engine = PolicyEngineUKEngine(clones=geographic_clones) contract = load_uk_take_up_contract() root_capture: dict[str, Frame] = {} root_transform = UKFRSSpineStageTransform(raw_dir, stage=stages["frs_spine"]) @@ -2432,6 +2465,15 @@ def capture_root(frame: Frame) -> Frame: "frs_annual_income_spells": UKFRSAnnualIncomeStageTransform( raw_dir, stage=stages["frs_annual_income_spells"] ), + # Every stage that reads the raw FRS tabs runs before the clones + # exist: geographic_expand gives every household geographic_clones + # clones, and geographic_identity keys each one (microcosm#931). + "geographic_expand": UKGeographicExpandStageTransform( + stage=stages["geographic_expand"], clones=geographic_clones + ), + "geographic_identity": UKGeographicIdentityStageTransform( + stage=stages["geographic_identity"] + ), "spi_support_channel": UKSPISupportChannelStageTransform( stage=stages["spi_support_channel"], sample_fraction=_SPI_SAMPLE_FRACTION, @@ -2548,7 +2590,7 @@ def _run_legacy_plan( stages: Iterable[SourceStageSpec], implementations: Mapping[str, object], ) -> Frame: - """Run the legacy 40-stage StagePlan oracle and return its final frame.""" + """Run the legacy StagePlan oracle and return its final frame.""" stages = tuple(stages) committed = load_country_spec("uk") @@ -2581,7 +2623,7 @@ def _run_legacy_plan( def legacy_oracle_frame(fixture: Path) -> Frame: """The legacy oracle's final frame on the fixture, computed live. - Rebuilds the same 31 transforms the graph's UK registry reconstructs from + Rebuilds the same transforms the graph's UK registry reconstructs from ``fixture/sources``, runs them through the legacy StagePlan in this process, root included, and applies the one string normalization the fixture documents. The content identity is a byte-exact fingerprint of @@ -2671,6 +2713,7 @@ def generate(output: Path) -> None: cgt_distribution=cgt_distribution, cgt_parameters=cgt_parameters, cgt_badr_parameters=cgt_badr_parameters, + geographic_clones=_GEOGRAPHIC_CLONES, ) # The PRIS facts are sized to the frame property_components reads and the # asset-type facts to the frame the amounts redraw and the anchor leave, @@ -2731,8 +2774,8 @@ def generate(output: Path) -> None: descriptor = { "schema_version": "uk-spine-parity-fixture.v1", "description": ( - "Data-only inputs for reconstructing the same 31 current UK stage " - "transform classes used by the legacy StagePlan oracle." + "Data-only inputs for reconstructing the current UK stage " + "transforms used by the legacy StagePlan oracle." ), "stages": {stage.stage: _stage_payload(stage) for stage in stages}, "config": { @@ -2740,6 +2783,7 @@ def generate(output: Path) -> None: "spi_donor_sample_size": _SPI_DONOR_SAMPLE_SIZE, "qrf_estimators": _QRF_ESTIMATORS, "student_loans_calibration_year": 2025, + "geographic_clones": _GEOGRAPHIC_CLONES, }, "inputs": { "frs_raw": "frs_raw", @@ -2767,7 +2811,7 @@ def generate(output: Path) -> None: } _write_json(sources / "fixture.json", descriptor) - graph = uk_spine_graph() + graph = uk_spine_graph(geographic_clones=_GEOGRAPHIC_CLONES) (output / "uk_spine.json").write_text(graph_to_json(graph) + "\n", encoding="utf-8") # Reported, never pinned: the identity is machine-specific (see # ``legacy_oracle_identity``), so the acceptance test computes it live. @@ -2779,8 +2823,8 @@ def generate(output: Path) -> None: _normalization_markdown(), encoding="utf-8" ) (output / "PRODUCED_BY.txt").write_text( - "tools/graph_uk_spine_fixture.py; current 31-transform legacy " - "StagePlan oracle with parsed private-source seams. The acceptance " + "tools/graph_uk_spine_fixture.py; current legacy StagePlan oracle " + "with parsed private-source seams. The acceptance " "test runs both sides from frs_raw in-process (root weights differ " "by one ulp between machines); the captured root tables serve the " "unbound registry's data-only CREATE path.\n", diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 7f36b441f9..bb1fe83deb 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -561,11 +561,14 @@ def e7_identity_receipt( """Receipt E7 deterministic layers under row permutation by entity id. Covered: the support-channel labelling the SPI stack introduces — each - entity's channel and clone index, the composite source key, and the - propagation of a household's channel down to its persons and benefit - units. These are pure functions of the synthetic flag and the entity ids, - so they are bitwise both under permutation and against the store, and they - are the layer E7 actually contributes to the artifact. + entity's channel and clone index, and the propagation of a household's + channel down to its persons and benefit units — and the composite source + key. These are pure functions of the synthetic flag, the entity ids and + each household's source year and source id, so they are bitwise both + under permutation and against the store. The labelling is the layer E7 + actually contributes to the artifact; the source key, which E7 wrote + before, is written by the clone-first spine's ``geographic_expand`` stage + (microcosm#931), and the receipt still checks it. Not covered here, and deliberately so: From 1c5440590938f22bcb4ada0c763bb089788a5190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:19:19 +0000 Subject: [PATCH 060/108] Receipt the geographic clones: fold variety, stage 16 and 17 forms, and clone-aware gate thresholds (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a spine of more than one geographic clone, the UK stage receipts now say how each FRS-trained fit's fold met its survey records. The SPI support channel and the income band donors count in source households, and the checks whose limits count rows read the clone count. The spine still builds one clone, so no value and no gate verdict moves. This follows María's rulings of 10 Nov: - decision 7: a threshold that grows with the clone count is multiplied by the receipt's clone_count, or counted in clone 0's rows; - decision 11: the band donors' funding rows take one form at every seating. Every new count is of distinct true source households, rounded through receipt_counts. Stage numbers here keep the order before stages 14 and 15 moved: 16 is spi_support_channel, 17 spi_income_band_donors, 18 hmrc_spi_income_spine and 25 regional_home_values. Fold receipts: - Every FRS-trained fit folds through household_lineage.fold_geographic_clones: the SPI income stage's reference and stage-2 forests, the SPI employment and housing shells, the UC reporter redraw, UC capital and salary sacrifice. - On more than one clone, each of these stages' evidence gains geographic_fold (household_lineage.geographic_fold_receipt). It gives the clone count, the source households the fit selected, those whose clones differ on what the fit reads, and the share that differ. The share is given only where ten or more of the selected source households differ and none or ten or more do not; otherwise the field repeats the count of those that differ. One clone's evidence is unchanged. - cross_fit_transport takes twin_identities, read only to find each recipient's twin among the training rows. The SPI income stage passes each copy's twin through its forests' fold, so the twin check reaches the copies of every clone. Each copy's uniforms stay keyed on its own identity; on one clone the two keys are the same. Stage 16 (spi_support_channel): the receipt gives the frame's clone count. Its two pension-age counts become rounded distinct source households carrying the mass beside them, on rows of positive weight: the pension-age base households', and those of clone 0's pension-age SPI copies. Both are the same at every clone count, and the masses are unchanged. Stage 17 (spi_income_band_donors): - At every seating, each funding row keeps its stratum and rounded donor households and reads "<3" for its masses and factor. The receipt gains two booleans: - funding_within_share: every factor is at least one less the declared reallocation share, to the stage check's relative tolerance; - funding_matches_bands: the bands and the funding strata each carry the reallocated mass. - On a capped seating, each band's donors and carriers, and the donor count, are rounded distinct source households. Each band's weighted taxpayers and carriers' mean age, and the reallocated mass, read "<3", and each band's carrier regions are left out. A full seating keeps the plan's exact band counts. - The result keeps the exact rows, and its frame, out of its repr. - A band short of candidates or seats is refused naming the band and, on a cloned spine, that a clone fell short, without a count. The band donor check: - It reads the funding from the two booleans and requires one less the receipt's reallocation share to reach its minimum funding factor. Its details drop the smallest funding factor. - On a capped seating it recomputes the seats from the transferred phase's frame. A seat is a band donor row whose id holds no ordinal in a copy-id field minted after the band donors. Every declared band must seat a donor, every seat must hold one carrier, and each band's seats must split evenly across the clones. The bands' mass there is the stage's funding_matches_bands, since later copy layers reweight the seats. The details then carry the clone count, the seating, the mass scale, the booleans and the count of bands. - No failure text gives a count, and a boolean that is present but not true is named as such. - The stage-health binding hands the frame to this check only (frame_predicate). copy_id_table joins the gate registry's modules, and the transferred phase's own code hashes it. The clone count: - geographic_clones.geographic_clone_count takes the calling stage and refuses a malformed clone index in fixed sentences naming it, with no count. The six support-clip stages, property components, the CGT support split and the fold receipts read K through it, and the two WAS stages bind the leaf for it. - GEOGRAPHIC_COPY_ID_FIELD, the copy-id field the clones' ids are minted in, is named in the leaf; geographic_expand re-exports it. Clone-aware thresholds: - The six support-clip stages (WAS wealth, WAS Lifetime ISA, NTS bus travel, LCFS consumption, ETB VAT and ETB services) and property components give clone_count when it is more than one. The support-clip check multiplies each declared row allowance by it, and the property components check its two row maxima. - Stage 25 (regional_home_values) counts the source households at the value cap in clone 0's rows, reading each row's clone from its id through the copy-id table, so its check's one maximum holds at any clone count. A frame whose clone index disagrees with its ids is refused. - The take-up signal receipt counts each flag's population support and distinct values, and the apprentice draw its supports, in clone 0's rows. A figure measured over every clone is withheld where either clone 0's count or the count over every clone is one or two. - test_uk_k_sensitive_constants.py classifies each count-like constant of the UK spine by what it counts and how it meets the clone count, and pins it to the code. Every count-like maximum in the packaged gates must be classified, the relationship grid's mismatch maximum among them. Text: changelog.d/931-uk-clone-receipts.changed.md is new. The unreleased band donor, CGT copy and fit fold fragments and docs/uk-atomic-household-lineage.md now describe the capped band counts, the count-free refusals and fold_geographic_clones as the fits' entry. What moves: - At K = 1 no value and no gate verdict moves. On synthetic frames this tree and its parent give the same frames and the same verdicts (pass or fail, and the failure count): the band donor check at full and capped seating, on a scaled mass, on a full sample seated capped and on a capped frame carried through every later copy layer, at one clone and at three. A one-clone receipt carries no clip or property clone_count, so those checks read their declared allowances as before, with the same texts and details. Every packaged clip and property allowance is zero, so the clone-scaled checks change no verdict at any clone count yet. - At K = 1 the bytes move only at stage 16 (clone_count and the two counts) and stage 17 (the funding rows and booleans at every seating, and the capped forms on a capped seating, as on the f010 and f001 samples), with the band donor check's details and failure texts and the clone count's refusal texts. The take-up signal and apprentice receipts count in clone 0's rows, which at one clone are every row. The geographic_fold receipts and the clip and property clone_count appear only from two clones up. - Node keys only: - Every UK spine node re-keys: graph_kernels' helper table re-keys every kernel it defines, the root and every counting stage hash household_lineage, and keys chain. - Both spine gate nodes re-key through stage_health and battery_bindings. The transferred phase's own code moves; the assembled phase's does not. - The full gates, preflight and terminal, re-key through battery_bindings, and the national gates through the registry's modules, copy_id_table now among them. The terminal phase's own code moves through etb_services, etb_vat, frs_take_up, lcfs_consumption and uc_capital_coherence; the preflight phase's does not. - The national problem and the target measures re-key, as do the dense population kernels (sample, normalize, expand, locations, geography mapping, geography gate and pool), the full build's identity, property value and private rent kernels, and the export preparation kernels, which hash the geographic clones' leaf. - 78 of the 105 UK kernel hashes and phase codes move (kernel.implementation_hash() against the parent). The other 27, among them the geography, export readback and package, target selection and compilation, problem, dense and size kernels, keep their own hashes, though their nodes re-key by chaining. - No generated file, declaration, fixture or gates.json changes, and H2 is not regenerated. No module outside the UK runtime imports an edited file, so no NZ, US, BE or transport key moves. Tests (engine-free; no engine test was needed): - The fold receipt is absent at one clone and rounded at three at all six sites, stage 18's through its checkpoint evidence. Frames with one and two varied sources give the same bytes, as three and nine do; ten varied sources give a share; all but one and all but two give the same bytes, as all but three and all but nine do; all varying give a share of 1. Every module whose fit folds builds the receipt. - On two clones, the twin check reads clone 1's copies against their folded twins and refuses a fold-key cell moved on one of them, while their uniforms stay their own. One clone is bitwise unchanged. - Stage 16's counts are the same at three and seven clones, count distinct sources at two, and leave out a row of zero weight. - Two capped stage 17 frames whose seats differ by one give byte-identical receipts, reprs and check results. - The band donor check fails a dropped carrier, an uneven split, a false, missing or non-boolean boolean, a looser share and a missing frame, with no data digit in any message. The transferred gate's inputs hold the cells it decodes. - Each support-clip stage, property components and regional home values run on three clones, with their checks unchanged at one; regional home values refuses a clone index that disagrees with the ids. The clone count's refusals are fixed sentences naming the stage. - The take-up and apprentice receipts count in clone 0 and withhold a figure that the other clones carry on one or two source households. - The constants table, and the binding, phase-code and copy-id tables. Co-Authored-By: Claude Opus 5.5 --- .../931-band-donors-per-clone.changed.md | 4 +- .../931-cgt-copies-source-weight.changed.md | 2 +- changelog.d/931-uk-clone-receipts.changed.md | 18 + ...31-uk-fits-fold-geographic-clones.added.md | 2 +- docs/uk-atomic-household-lineage.md | 2 +- .../build/uk_runtime/battery_bindings.py | 15 + .../microcosm/build/uk_runtime/cgt_support.py | 4 +- .../build/uk_runtime/etb_services.py | 15 +- .../src/microcosm/build/uk_runtime/etb_vat.py | 12 +- .../build/uk_runtime/frs_person_draws.py | 55 +- .../microcosm/build/uk_runtime/frs_take_up.py | 88 ++- .../build/uk_runtime/gate_phase_codes.py | 21 +- .../build/uk_runtime/geographic_clones.py | 62 +- .../build/uk_runtime/geographic_expand.py | 6 +- .../build/uk_runtime/graph_kernels.py | 10 +- .../build/uk_runtime/household_lineage.py | 80 +- .../build/uk_runtime/lcfs_consumption.py | 10 +- .../build/uk_runtime/nts_bus_travel.py | 4 + .../build/uk_runtime/property_components.py | 14 +- .../build/uk_runtime/regional_home_values.py | 33 +- .../build/uk_runtime/salary_sacrifice.py | 33 +- .../build/uk_runtime/spi_band_donors.py | 218 +++++- .../build/uk_runtime/spi_employment_shell.py | 26 +- .../build/uk_runtime/spi_housing_shell.py | 27 +- .../microcosm/build/uk_runtime/spi_income.py | 75 +- .../build/uk_runtime/spi_rank_transport.py | 21 +- .../microcosm/build/uk_runtime/spi_spine.py | 43 +- .../microcosm/build/uk_runtime/spi_support.py | 46 +- .../build/uk_runtime/stage_health.py | 424 ++++++++--- .../build/uk_runtime/support_clip.py | 38 +- .../build/uk_runtime/uc_capital_coherence.py | 35 +- .../build/uk_runtime/uc_reporter_redraw.py | 27 +- .../microcosm/build/uk_runtime/was_lisa.py | 4 + .../microcosm/build/uk_runtime/was_wealth.py | 18 +- .../engine_free/uk/test_uk_apprentice_draw.py | 95 +++ .../uk/test_uk_battery_bindings.py | 142 ++++ .../engine_free/uk/test_uk_cgt_support.py | 49 +- .../engine_free/uk/test_uk_copy_id_table.py | 25 +- .../engine_free/uk/test_uk_etb_services.py | 77 ++ .../tests/engine_free/uk/test_uk_etb_vat.py | 47 ++ .../uk/test_uk_geographic_clone_collapse.py | 179 +++++ .../tests/engine_free/uk/test_uk_graph.py | 32 +- .../engine_free/uk/test_uk_graph_evidence.py | 16 + .../uk/test_uk_k_sensitive_constants.py | 522 +++++++++++++ .../uk/test_uk_lcfs_consumption.py | 25 + .../engine_free/uk/test_uk_nts_bus_travel.py | 49 ++ .../uk/test_uk_property_components.py | 63 ++ .../uk/test_uk_regional_home_values.py | 190 +++++ .../uk/test_uk_salary_sacrifice.py | 27 + .../engine_free/uk/test_uk_spi_band_donors.py | 358 ++++++++- .../uk/test_uk_spi_employment_shell.py | 37 + .../uk/test_uk_spi_housing_shell.py | 38 + .../engine_free/uk/test_uk_spi_income.py | 190 +++++ .../uk/test_uk_spi_rank_transport.py | 128 ++++ .../tests/engine_free/uk/test_uk_spi_spine.py | 169 +++++ .../engine_free/uk/test_uk_spi_support.py | 113 +++ .../engine_free/uk/test_uk_stage_health.py | 692 ++++++++++++++++-- .../engine_free/uk/test_uk_take_up_gate.py | 72 ++ .../uk/test_uk_uc_capital_coherence.py | 30 + .../uk/test_uk_uc_reporter_redraw.py | 71 ++ .../tests/engine_free/uk/test_uk_was_lisa.py | 35 + .../engine_free/uk/test_uk_was_wealth.py | 38 + 62 files changed, 4591 insertions(+), 410 deletions(-) create mode 100644 changelog.d/931-uk-clone-receipts.changed.md create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_k_sensitive_constants.py diff --git a/changelog.d/931-band-donors-per-clone.changed.md b/changelog.d/931-band-donors-per-clone.changed.md index 31777f4114..a4ddff4034 100644 --- a/changelog.d/931-band-donors-per-clone.changed.md +++ b/changelog.d/931-band-donors-per-clone.changed.md @@ -1,7 +1,7 @@ The SPI income band donors (`spi_income_band_donors`) are seated within each geographic clone (#931). Where the households carry `household_clone_index`, the stage runs its Efraimidis-Spirakis walk once in each clone, over that clone's candidates, with the same person-keyed uniforms, keys and tie order, and the same seats in every clone (María's rulings of 7 Nov). The seats and the household share's cap read the eligible households counted in distinct `source_household_id` values, so they do not depend on the clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, seats exactly the one-clone donors, carriers and bands. Each donor weighs its band's published taxpayers over the clone count times one clone's band donor count. A spine without the column is one clone; the spine's `geographic_expand` stage writes it, 0 on every household of a one-clone spine. - **Funding.** A donor's funding stratum is its region within its clone, a derived stratum (the manifest's `funding_strata` stays `[region]`), so every clone's regions conserve their own mass. One mass scale, the smallest any clone's region allows, serves every clone, so each band keeps one donor weight and the stage gate's and the identity tool's band-mass reconciliations hold as before. The exact-total correction still lands on one incumbent row, never on a donor, so a band's donors stay equal bit for bit at any clone count. The seating nests exactly at any clone count. Clone 0's weights and funding factors nest, its weights as the one-clone weights over the clone count, only while the common scale is the one a one-clone spine takes: one at both, or bound by a region of clone 0. Where another clone's region binds, every clone's donors, clone 0's included, take the lower scale. Even while they nest, the weights nest to rounding, not bitwise: the corrected row, which can sit in any clone, moves by a few ulps of the total, and the 1/K split rounds when the clone count is not a power of two, so uncorrected rows also miss by about one ulp then. The mass record's reason is unchanged, so neither the frame's content identity nor the coverage manifest's restated reason moves. -- **Receipt.** The evidence gains `clone_count`, and `funding_strata` lists `household_clone_index` where the column is present (the funding rows' labels then read `region|clone`). `expected_donor_households`, `expected_donor_count` and the donor counts cover every clone; `donor_weight` and `planned_donor_weight` are one clone's weights. Under the receipts' floor of three (`receipt_counts`), the eligible household counts, in all and per band, are counted in source households and rounded in place of the exact counts, each band's being clone 0's, which a one-clone spine counts alike; `seating_scale` reads `full` or `capped`, since an exact scale below one gives back the eligible count; a band's `carrier_mean_age` reads `<3` when its carriers come from fewer than three source households or the band seats fewer than three households per clone; and `carrier_region_counts` and the funding rows' `donor_households` are rounded. The band rows' donor and carrier counts stay exact row counts, which the gate reads. +- **Receipt.** The evidence gains `clone_count`, and `funding_strata` lists `household_clone_index` where the column is present (the funding rows' labels then read `region|clone`). `expected_donor_households`, `expected_donor_count` and the donor counts cover every clone; `donor_weight` and `planned_donor_weight` are one clone's weights. Under the receipts' floor of three (`receipt_counts`), the eligible household counts, in all and per band, are counted in source households and rounded in place of the exact counts, each band's being clone 0's, which a one-clone spine counts alike; `seating_scale` reads `full` or `capped`, since an exact scale below one gives back the eligible count; a band's `carrier_mean_age` reads `<3` when its carriers come from fewer than three source households or the band seats fewer than three households per clone; and `carrier_region_counts` and the funding rows' `donor_households` are rounded. At full seating the band rows' donor and carrier counts stay exact row counts, which the gate reads; on a capped seating they are rounded source households, and the gate recomputes the seats from the frame. - **Gate.** The `spi_income_band_donor_support` check refuses a receipt without a positive integer `clone_count`. A donor's weight times the clone count must stay within the maximum, a band's donors must split evenly across the clones, and at full seating a band seats its plan and never fewer than the clone count times the minimum. The seating scale is read as `full` or `capped`, and a value in neither form is named by its type alone. At one clone the checks and their messages are as before, apart from the seating scale's form. -- **Refusals.** A clone short of candidates for a band is refused with the number of such clones and the fewest candidates found. A non-integer, missing or negative clone index, or an index from 0 to K - 1 that no household carries, is refused by count. +- **Refusals.** A clone short of candidates for a band is refused naming the band, without a count. A non-integer, missing or negative clone index, or an index from 0 to K - 1 that no household carries, is refused by count. - **What moves.** At K = 1, no value: on synthetic frames the seating, the carriers, every weight, the funding factors, the mass and seating scales, the mass record and the frame's content identity after the stage are bit for bit the parent commit's, with or without a one-clone index. The receipt takes the forms above. The stage's draw, weight-rule, funding, seating-scale and mass-scale texts and its notes change in `sources.yaml`, `source_stages.json` and the stage dictionary, so the stage's contract hash, the H2 fixture's declarations and the coverage manifest's 25 source digests are regenerated. `spi_income_band_donors` and every node after it re-key, as do the spine gates and the stages that bind `stage_health`. diff --git a/changelog.d/931-cgt-copies-source-weight.changed.md b/changelog.d/931-cgt-copies-source-weight.changed.md index a182eedc5e..84c459b7c4 100644 --- a/changelog.d/931-cgt-copies-source-weight.changed.md +++ b/changelog.d/931-cgt-copies-source-weight.changed.md @@ -1,4 +1,4 @@ -The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, which the spine's `geographic_expand` stage writes, or 1 where the column is absent. It refuses, by count, indices that are not exactly 0 to K − 1. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. +The CGT support split (`cgt_support_split`) counts each selected household's copies on its weight scaled to its source, `ceil(K × w / 60)`, where `K` is the frame's geographic clone count (#931). A geographic clone carries 1/K of its source household's weight, so a selected clone whose weight is an exact 1/K replica of its source's (which the per-clone strata of the later #931 commits give) gets the count its source gets on a one-clone spine, and each copy weighs at most 60 / K. The stage reads `K` from the frame: one more than the largest `household_clone_index`, which the spine's `geographic_expand` stage writes, or 1 where the column is absent. It refuses indices that are not exactly 0 to K − 1 in a fixed sentence naming the stage, with no count. The selection walk still accumulates each row's own weight, so selection nests only approximately: the household at which a band's walk reaches its support mass, and households tied on wealth, can be selected in some clones and not others, lowest household id first. For K > 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. - **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: for a weight a few ulps above a multiple of 60 (at K > 1, by more than the snap absorbs), or where an exact-total correction or the #970 anchor's and the residential arms' roundings moved a member. So neither the count check nor the family test recounts the fold. At any K, a family of more than three members is held exactly where at least four of its members show their two clone halves at equal weights and every member that does shares one pair sum c, whatever the others carry: its stored count n must be the count of a weight w with fl(w / n) = c. A member that took the split's own fallback correction keeps equal halves at another weight, since the incidence clone halves it exactly, so its family keeps the bracket. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so one of those four or more is uncorrected, and equal halves mean the anchor and the arms left its pair sum at exactly w / n, so c is exactly w / n. The stage counts n on each clone's own weight and gives every member fl(w / n), and the snapped count still rises with w, so this holds at K > 1 as at K = 1. Where every member shares c but fewer than four show equal halves, the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member where the floats either side of c are as wide, and two floats where c sits within two floats of a power of two, below which they are half as wide; c lies in (45 / K, 60 / K] to within the snap, which can leave it a few floats above 60 / K (at most 16 for K up to 15, by design arithmetic over weights at and up to 40 floats above each multiple m × 60 / K, m from 4 to 3,999), and that range holds 16 at K = 3, 8 at K = 6 and 7, and 4 at K = 12 to 15. Every other family keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. The family test, which decides whether a household is a family at all, is held the same way at the K-aware count: a household the rule selects is possibly a family where its count at the folded weight times 1 + 1e-9 exceeds one and possibly whole where its count at the folded weight times 1 − 1e-9 is one, its stored state accepted in between, and the hold's counts decide where it applies. That bracket opens for a household whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies, a whole household or a family of two among them. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a household the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to a few floats next to the count's step, for some counts n. At K = 1 the step sits a float above the multiple, which leaves at most the multiple itself where the family is held exactly, or the multiple and the float either side of it where the hold reaches one float. At K from 2 to 15 the snap puts the step 3 to 17 floats above the multiple of 60 / K, which leaves at most the float on either side of the step where the family is held exactly, or up to three floats from it in the one-float tier, so a family on the multiple itself split into one copy more is reported. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. - **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception and no more, since it makes no one-copy check. Both stages' notes say where the correction lands, and the CGT stage's add that the shared fallback does not make the one-copy check, so it can put the correction on a one-copy household the check passed over, and that the identity tool's selection recompute can then count that household as a family the stage never made, which the bracketed family test does not report, or miss one it made, which the tool reports. Such a household, lifted across 60 / K, moves only `copies_recomputed`, since the bracketed family test cannot tell it from a whole household (see Receipt and gate). The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. diff --git a/changelog.d/931-uk-clone-receipts.changed.md b/changelog.d/931-uk-clone-receipts.changed.md new file mode 100644 index 0000000000..c4caf5e99d --- /dev/null +++ b/changelog.d/931-uk-clone-receipts.changed.md @@ -0,0 +1,18 @@ +The UK stage receipts and stage checks now read a spine of more than one geographic clone (#931; María's rulings of 10 Nov). Every new count is of distinct true source households, rounded through `receipt_counts`. The spine still builds one clone, so no value and no gate verdict moves. At one clone only the `spi_support_channel` and `spi_income_band_donors` receipts change, with the band donor check's details and failure texts and the clone count's refusal texts. + +- **Fold receipts.** Every FRS-trained fit folds its training rows through `household_lineage.fold_geographic_clones`: the SPI income stage's reference and stage-2 forests, the SPI employment and housing shells, the UC reporter redraw, UC capital and salary sacrifice. On more than one clone, each of these stages' evidence gains `geographic_fold` (`household_lineage.geographic_fold_receipt`). It gives the clone count; the source households the fit selected, and those whose clones differ on what the fit reads, both rounded as every receipt count is; and the share that differ, given only where ten or more of the selected households differ and none or ten or more do not, and otherwise the count of those that differ again. A one-clone spine's evidence is unchanged. +- **Twin check.** `cross_fit_transport` takes `twin_identities`, read only to find each recipient's twin among the training rows. The SPI income stage passes each copy's twin through its forests' fold, so the check reaches the copies of every clone. Each copy's uniforms stay keyed on its own identity; on one clone the two keys are the same. +- **`spi_support_channel`.** Its receipt gives the frame's clone count. Its two pension-age counts are rounded distinct source households carrying the mass beside them, on rows of positive weight: the pension-age base households', and those of clone 0's pension-age SPI copies. Both are the same at every clone count, and the masses are unchanged. +- **`spi_income_band_donors`.** + - At every seating, each funding row keeps its stratum and its rounded donor households and reads `<3` for its masses and factor. The receipt gains two booleans: `funding_within_share`, every factor at least one less the declared reallocation share, to the stage check's relative tolerance; and `funding_matches_bands`, the bands and the funding strata each carrying the reallocated mass. + - On a capped seating, each band's donors and carriers, and the donor count, are rounded distinct source households. Each band's weighted taxpayers and carriers' mean age, and the reallocated mass, read `<3`, and each band's carrier regions are left out. A full seating keeps the plan's exact band counts. + - The result keeps the exact figures, and its frame, out of its repr. + - A band short of candidates or seats is refused naming the band and, on a cloned spine, that a clone fell short, without a count. +- **The band donor check.** It reads the funding from the two booleans and requires one less the receipt's reallocation share to reach its minimum funding factor; its details drop the smallest funding factor. On a capped seating it recomputes the seats from the transferred phase's frame. A seat is a band donor row whose id holds no ordinal in a copy-id field minted after the band donors, since every later layer keeps the row it copies at its own id. Every declared band must seat a donor, every seat must hold one carrier, and each band's seats must split evenly across the clones; the bands' mass there is the stage's `funding_matches_bands`, since later layers reweight the seats. The details then carry the clone count, the seating, the mass scale, the booleans and the count of bands. No failure text gives a count. The stage-health binding hands the frame to this check alone, `copy_id_table` joins the gate registry's modules, and the transferred phase's own code hashes it. +- **Clone count.** `geographic_clones.geographic_clone_count` takes the calling stage and refuses a malformed clone index in fixed sentences naming it, with no count. The six support-clip stages, property components, the CGT support split and the fold receipts read the count through it, and the WAS wealth and Lifetime ISA stages bind the leaf for it. The copy-id field the clones' ids are minted in, `GEOGRAPHIC_COPY_ID_FIELD`, is named in the leaf, and `geographic_expand` re-exports it. +- **Clone-aware thresholds.** + - The six support-clip stages (WAS wealth, WAS Lifetime ISA, NTS bus travel, LCFS consumption, ETB VAT and ETB services) and property components give `clone_count` from two clones up. The support-clip check multiplies each declared row allowance by it, and the property components check its two row maxima. + - `regional_home_values` counts the source households at the value cap in clone 0's rows, reading each row's clone from its id through the copy-id table, so its check's one maximum holds at any clone count. It refuses a frame whose clone index disagrees with its ids. + - The take-up signal receipt counts each flag's population support and distinct values in clone 0's rows, and the apprentice draw its supports. A figure measured over every clone is withheld where clone 0's count or the count over every clone is one or two. +- **Constants table.** `test_uk_k_sensitive_constants.py` classifies each count-like constant of the UK spine, and each count-like maximum in the packaged gates, by what it counts and how it meets the clone count, and pins it to the code. +- **What moves.** Node keys only. Every UK spine node re-keys. On a bound spine, so do the full build's identity, property value and private rent kernels, the dense population kernels, the export preparation kernels, the national problem, measures and gates, and the full gates with their terminal phase code. No generated file, declaration or `gates.json` changes. No transport kernel and no US, NZ or BE node hashes an edited module; only the US PUF-support builder's whole-tree resume fingerprint moves, as on any edit to packaged source. diff --git a/changelog.d/931-uk-fits-fold-geographic-clones.added.md b/changelog.d/931-uk-fits-fold-geographic-clones.added.md index 9883d358aa..64149fafc4 100644 --- a/changelog.d/931-uk-fits-fold-geographic-clones.added.md +++ b/changelog.d/931-uk-fits-fold-geographic-clones.added.md @@ -10,7 +10,7 @@ The UK fits trained on the frame's FRS rows now fold geographic clones into each - `copy_paths`: every household's ordered copy path from a declared lineage, never inferred from ids, weights or values; - `replica_keys`: a key per record of any entity, shared only by its replicas in the named branches, with a member's rank in its household telling members apart; - `collapse_replicas`: the fold's training rows and weights (`fold_replicas` returns the whole fold), folding only replicas that agree on the fit's `inputs`. -- **The UK entry.** `household_lineage.collapse_geographic_clones` is the UK's entry and requires those inputs. `uk_household_clone_paths` now reads the clone path through `copy_paths` on `UK_COPY_LINEAGE`, with identical paths and draw keys; three identity-test messages name the shared reader's branches. +- **The UK entry.** `household_lineage.fold_geographic_clones` is the UK's entry and requires those inputs; `collapse_geographic_clones` returns its rows and weights. `uk_household_clone_paths` now reads the clone path through `copy_paths` on `UK_COPY_LINEAGE`, with identical paths and draw keys; three identity-test messages name the shared reader's branches. - **Declarations and tests.** - The collapsing stages declare the lineage live where they run, and a test refuses one whose surface would hide it. - Replicated-fixture tests hold each fit to the rows and weights it fits without clones. diff --git a/docs/uk-atomic-household-lineage.md b/docs/uk-atomic-household-lineage.md index 101ff7e544..dd9951f61a 100644 --- a/docs/uk-atomic-household-lineage.md +++ b/docs/uk-atomic-household-lineage.md @@ -29,7 +29,7 @@ The UK's copy lineage is declared once, as data, in `uk_runtime.household_lineag - `record_sources` gives every record of any entity its true source through household membership. Every UK floor counts distinct sources through it (`receipt_counts.record_source_households`), so a household's copies and clones count once. - `copy_paths` reads every household's ordered clone path; `uk_household_clone_paths` is the UK's strict entry for the identity kernel. -- `replica_keys` and `fold_replicas` keep one training row per true source and path without the geographic branch, with the clones' weights summed; through the UK entries each variant trains on its lowest-id clone, whatever the row order. `collapse_replicas` returns the fold's rows and weights alone. The FRS-trained fits fold their training rows through the UK entries (`household_lineage.collapse_geographic_clones`, or `fold_geographic_clones` where a stage reads the whole fold, as UC capital does to count its cells' donors in survey records): the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. Once geographic clones precede enrichment, each fit still sees each survey record once. A frame without geographic clones keeps its rows and weights value for value. +- `replica_keys` and `fold_replicas` keep one training row per true source and path without the geographic branch, with the clones' weights summed; through the UK entries each variant trains on its lowest-id clone, whatever the row order. `collapse_replicas` returns the fold's rows and weights alone. The FRS-trained fits fold their training rows through the UK entry (`household_lineage.fold_geographic_clones`; `collapse_geographic_clones` returns its rows and weights alone): the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. UC capital also counts its cells' donors in survey records. On a frame of more than one geographic clone, each of these stages records the fold in its evidence as `geographic_fold` (`household_lineage.geographic_fold_receipt`): the clone count, the true source households the fit selected and those whose clones differ on what it reads, rounded as every receipt count is, and the share that differ, given only where ten or more of the selected sources differ and none or ten or more do not. Once geographic clones precede enrichment, each fit still sees each survey record once. A frame without geographic clones keeps its rows and weights value for value. - Each fit passes what it reads (its predictors and targets, or the columns they derive from), and clones fold only where they agree on it. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own. Each variant carries the summed weight of the clones that share it, so the fit sees every clone's value at its share and nothing is dropped. ## Verification scope diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 1b37ad7bc6..76b3f0ac84 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -323,6 +323,17 @@ def _evaluate_calibration_reference_coverage( ) +#: The stage-health checks that read the phase's frame beside their stage's +#: receipt: the band donors' check recomputes a capped seating's seats from +#: the transferred phase's frame (microcosm#931). Every other check reads its +#: receipt alone and runs without a frame. +_STAGE_HEALTH_FRAME_CHECKS = frozenset({"spi_income_band_donor_support"}) + + +def _stage_health_requires_frame(parameters: Mapping[str, Any]) -> bool: + return parameters.get("check") in _STAGE_HEALTH_FRAME_CHECKS + + def _evaluate_stage_health( context: EvidenceContext, parameters: Mapping[str, Any] ) -> GateResult: @@ -338,6 +349,7 @@ def _evaluate_stage_health( stage=stage, check=str(parameters["check"]), parameters=parameters, + frame=context.frame if _stage_health_requires_frame(parameters) else None, ) @@ -2283,6 +2295,8 @@ def _ledger_compile_parity_required_artifacts( ), artifact_keys=frozenset({"stage_evidence"}), needs_frame=False, + # The band donors' check reads the phase's frame (microcosm#931). + frame_predicate=_stage_health_requires_frame, evidence=_stage_health_evidence, ), "nonnegative_columns": UKGateBinding( @@ -2491,6 +2505,7 @@ def _ledger_compile_parity_required_artifacts( "microcosm.build.uk_runtime.bus_use_incidence", "microcosm.build.uk_runtime.cgt_projection", "microcosm.build.uk_runtime.chronicle_feed", + "microcosm.build.uk_runtime.copy_id_table", "microcosm.build.uk_runtime.council_tax_constants", "microcosm.build.uk_runtime.dfe_spring_term", "microcosm.build.uk_runtime.diagnostics", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index 6f92b485aa..cf4ddbdef4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -291,7 +291,9 @@ def __call__(self, frame: Frame) -> Frame: frame, distribution=distribution, parameters=parameters, - clone_count=geographic_clone_count(frame.table("household")), + clone_count=geographic_clone_count( + frame.table("household"), stage=CGT_SUPPORT_SPLIT_STAGE_NAME + ), ) object.__setattr__(self, "last_result", result) return result.frame diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index b9ea27b057..6fde51c5e8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -29,6 +29,7 @@ ) from microcosm.build.uk_runtime.etb_vat import UK_ETB_FAMILY_ROLE_COUNTS from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_NET_INCOME_PREDICTOR, UK_HBAI_NET_INCOME_VARIABLE, @@ -211,7 +212,12 @@ def __call__(self, frame: Frame) -> Frame: outcomes=outcomes, ) clip_result = support_clip_to_donor( - draws, donor, exempt=support_clip_exempt(self.stage) + draws, + donor, + exempt=support_clip_exempt(self.stage), + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_ETB_SERVICES_STAGE_NAME + ), ) draws = clip_result.clipped household = frame.table("household").copy() @@ -519,7 +525,11 @@ def impute_etb_services( def support_clip_to_donor( - draws: pd.DataFrame, donor: pd.DataFrame, *, exempt: set[str] | None = None + draws: pd.DataFrame, + donor: pd.DataFrame, + *, + exempt: set[str] | None = None, + clone_count: int = 1, ) -> UKSupportClipResult: return support_clip_to_donor_with_receipt( draws, @@ -527,6 +537,7 @@ def support_clip_to_donor( columns=UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS[:3], stage=UK_ETB_SERVICES_STAGE_NAME, exempt=exempt, + clone_count=clone_count, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py index d294a5d81d..87e864717d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -19,6 +19,7 @@ uprating_operation, ) from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_GROSS_INCOME_VARIABLES, etb_disposable_income, @@ -147,7 +148,13 @@ def __call__(self, frame: Frame) -> Frame: household_ids=frame.table("household")["household_id"].to_numpy(), salt=_qrf_salt(self.stage), ) - clip_result = support_clip_to_donor(imputed, donor) + clip_result = support_clip_to_donor( + imputed, + donor, + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_ETB_VAT_STAGE_NAME + ), + ) imputed = clip_result.clipped household = frame.table("household").copy() household["full_rate_vat_expenditure_rate"] = imputed[ @@ -398,13 +405,14 @@ def impute_etb_vat( def support_clip_to_donor( - draws: pd.DataFrame, donor: pd.DataFrame + draws: pd.DataFrame, donor: pd.DataFrame, *, clone_count: int = 1 ) -> UKSupportClipResult: return support_clip_to_donor_with_receipt( draws, donor, columns=UK_ETB_VAT_OUTPUT_COLUMNS, stage=UK_ETB_VAT_STAGE_NAME, + clone_count=clone_count, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py index f445f3a33d..b9a4ed6516 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py @@ -50,6 +50,7 @@ ) from microcosm.build.uk_runtime.frs_employment import FRS_EMPSTATI_EMPLOYMENT_STATUS from microcosm.build.uk_runtime.frs_take_up import ( + geographic_clone_zero_rows, take_up_signal_outputs, uk_take_up_signal_receipt, ) @@ -186,6 +187,7 @@ def add_frs_person_draws_with_receipt( person_sources=record_source_households( frame, "person", stage="frs_person_draws", before_copies=True ), + person_clone_zero=geographic_clone_zero_rows(frame, "person"), ) for column in FRS_PERSON_DRAW_OUTPUT_COLUMNS: person[column] = derived[column].to_numpy() @@ -209,12 +211,14 @@ def derive_frs_person_draws( person_weights: np.ndarray, relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, + person_clone_zero: np.ndarray | None = None, ) -> pd.DataFrame: """Return the stage's person draws. ``person_weights`` are each person's household weight, which set ``is_apprentice``'s target; ``person_sources`` are each person's true - source household, which its receipt counts support by. + source household, which its receipt counts support by, among geographic + clone 0's people (``person_clone_zero``; everyone when not given). """ return derive_frs_person_draws_with_receipt( @@ -223,6 +227,7 @@ def derive_frs_person_draws( person_weights=person_weights, relief_schedule=relief_schedule, person_sources=person_sources, + person_clone_zero=person_clone_zero, )[0] @@ -233,6 +238,7 @@ def derive_frs_person_draws_with_receipt( person_weights: np.ndarray, relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, + person_clone_zero: np.ndarray | None = None, ) -> tuple[pd.DataFrame, Mapping[str, object]]: """The stage's person draws and the receipt of its ``is_apprentice`` draw.""" @@ -266,6 +272,7 @@ def derive_frs_person_draws_with_receipt( person_weights=person_weights, relief_schedule=relief_schedule, person_sources=person_sources, + person_clone_zero=person_clone_zero, ) values[APPRENTICE_OUTPUT] = apprentices.flags return values, {"build_year": contract.build_year, **apprentices.receipt} @@ -286,6 +293,7 @@ def derive_is_apprentice( person_weights: np.ndarray | None = None, relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, + person_clone_zero: np.ndarray | None = None, ) -> UKApprenticeDraw: """Draw ``is_apprentice`` for employees aged 21 to 24 outside full-time education. @@ -307,7 +315,14 @@ def derive_is_apprentice( The receipt counts support in ``person_sources``, each person's true source household (``receipt_counts.record_source_households``). A caller without the frame's lineage passes ``own_source_households`` itself, so - no table is counted by its rows unawares (microcosm#931). + no table is counted by its rows unawares (microcosm#931). The supports + it prints are counted among geographic clone 0's people, + ``person_clone_zero`` (everyone when not given): each clone flags its own + employees, so the flagged sources over every clone would grow with the + clone count. The masses are taken over every clone, so each is withheld + where one or two source households carry it in clone 0's people or over + every clone: clone 0 can flag none of the employees one or two + households hold in the other clones. """ if "employment_status" not in person.columns: @@ -353,9 +368,27 @@ def derive_is_apprentice( sources = np.asarray(person_sources) if sources.shape != (len(person),): raise ValueError("person_sources must hold one source per person.") - population_support = distinct_count(sources, in_ages, weights) - eligible_support = distinct_count(sources, eligible, weights) - flagged_support = distinct_count(sources, flags, weights) + clone_zero = ( + np.ones(len(person), dtype=bool) + if person_clone_zero is None + else np.asarray(person_clone_zero, dtype=bool) + ) + if clone_zero.shape != (len(person),): + raise ValueError("person_clone_zero must hold one flag per person.") + population_support = distinct_count(sources, in_ages & clone_zero, weights) + eligible_support = distinct_count(sources, eligible & clone_zero, weights) + flagged_support = distinct_count(sources, flags & clone_zero, weights) + # The masses are taken over every clone, so a mass is withheld where one + # or two source households carry it in clone 0 or over every clone. + population_withheld = _thin(population_support) or _thin( + distinct_count(sources, in_ages, weights) + ) + eligible_withheld = _thin(eligible_support) or _thin( + distinct_count(sources, eligible, weights) + ) + flagged_withheld = _thin(flagged_support) or _thin( + distinct_count(sources, flags, weights) + ) receipt: dict[str, object] = { "output": APPRENTICE_OUTPUT, "rate_key": APPRENTICE_RATE_KEY, @@ -376,13 +409,13 @@ def derive_is_apprentice( # A weighted total carried by one or two households is withheld, with the # figures that would give it back: the target is the rate times the # population, and the deviation is the flagged weight less the target. - if 0 < population_support < UK_DISCLOSURE_FLOOR: + if population_withheld: receipt.update( population_mass=UK_WITHHELD, target=UK_WITHHELD, realization_deviation=UK_WITHHELD, ) - if 0 < eligible_support < UK_DISCLOSURE_FLOOR: + if eligible_withheld: receipt["eligible_mass"] = UK_WITHHELD if relief_schedule is not None and "employment_income" in person.columns: pay = pd.to_numeric(person["employment_income"], errors="coerce") @@ -399,13 +432,19 @@ def derive_is_apprentice( receipt["implied_relief"] = float( relief_schedule["rate"] * (weights[flags] * relieved[flags]).sum() ) - if 0 < flagged_support < UK_DISCLOSURE_FLOOR: + if flagged_withheld: receipt.update(realized_mass=UK_WITHHELD, realization_deviation=UK_WITHHELD) if "implied_relief" in receipt: receipt["implied_relief"] = UK_WITHHELD return UKApprenticeDraw(flags=flags, receipt=receipt) +def _thin(support: int) -> bool: + """Whether one or two source households carry a figure: it is withheld.""" + + return 0 < support < UK_DISCLOSURE_FLOOR + + def _nearest_target_threshold( need: float, *, draws: np.ndarray, weights: np.ndarray ) -> float: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py index 70cb2da3a6..4ea3c505ef 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py @@ -19,13 +19,16 @@ stable_identity_uniforms, ) from microcosm.build.uk_runtime.dfe_spring_term import dfe_age_at_31_december +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, uk_household_weight_kind, uk_national_frame, uk_time_period, validate_uk_national_frame, ) from microcosm.build.uk_runtime.receipt_counts import ( + UK_DISCLOSURE_FLOOR, distinct_count, extreme_count, record_source_households, @@ -730,6 +733,49 @@ def uk_take_up_signal_gate( ) +def geographic_clone_zero_rows(frame: Frame, entity: str) -> np.ndarray: + """Whether each ``entity`` row belongs to geographic clone 0 (microcosm#931). + + A row's clone is its household's ``household_clone_index``: a person's + household, or the household a benefit unit's members share. A frame + without the index is one clone, every row clone 0's. A count a receipt + takes in clone 0's rows does not grow with the clone count, and on a + one-clone spine it is the count over every row. + """ + + household = frame.table("household") + table = frame.table(entity) + column = ladder_clone_index_column("household") + if column not in household.columns: + return np.ones(len(table), dtype=bool) + zero = pd.Series( + household[column].to_numpy() == 0, + index=household[UK_NATIONAL_SCHEMA.id_column("household")].to_numpy(), + ) + if entity == "household": + return zero.to_numpy(dtype=bool) + person = frame.table(UK_NATIONAL_SCHEMA.person_entity) + person_zero = person[UK_NATIONAL_SCHEMA.membership_column("household")].map(zero) + if person_zero.isna().any(): + raise ValueError( + "Every person needs a household on the frame to read its geographic clone." + ) + if entity == UK_NATIONAL_SCHEMA.person_entity: + return person_zero.to_numpy(dtype=bool) + by_unit = ( + pd.Series( + person_zero.to_numpy(dtype=bool), + index=person[UK_NATIONAL_SCHEMA.membership_column(entity)].to_numpy(), + ) + .groupby(level=0) + .first() + ) + unit_zero = table[UK_NATIONAL_SCHEMA.id_column(entity)].map(by_unit) + if unit_zero.isna().any(): + raise ValueError(f"A {entity} without members has no geographic clone to read.") + return unit_zero.to_numpy(dtype=bool) + + @dataclass(frozen=True) class _TakeUpSignal: """One flag's weighted share over its draw's population.""" @@ -748,14 +794,21 @@ def _measure_take_up_signal( output: str, key: str, contract: UKTakeUpContract, + clone_zero: np.ndarray | None = None, ) -> _TakeUpSignal | str: - """One flag's share at the frame's weights, or why it cannot be measured.""" + """One flag's share at the frame's weights, or why it cannot be measured. + + The flag's distinct values are counted in geographic clone 0's rows, + ``clone_zero``, as one clone counts them (microcosm#931); without it, in + every row, as the terminal gate counts them. + """ table = frame.table(entity) if output not in table.columns: return "missing stochastic column." values = np.asarray(table[output], dtype=bool) - unique_count = int(pd.Series(values).nunique(dropna=False)) + counted = values if clone_zero is None else values[clone_zero] + unique_count = int(pd.Series(counted).nunique(dropna=False)) weights = np.asarray(frame.resolve_weights(entity).values, dtype=np.float64) population = np.ones(values.shape, dtype=bool) if key == UK_UNDER_TWO_USAGE_SIGNAL_KEY: @@ -818,15 +871,29 @@ def uk_take_up_signal_receipt( 2026-10-29, microcosm#1095). The receipt is aggregate-only (UKDS disclosure control): a share or a target is given only where three or more source households carry the flag's population, whose support is a band. + + The population's support and the flag's distinct values are counted in + geographic clone 0's rows (:func:`geographic_clone_zero_rows`; + microcosm#931): each clone draws its own flags, and the under-2 usage + population with them, so a count over every clone would grow with the + clone count. The share and the target are measured over every clone, so + each is withheld where one or two source households carry the population + in clone 0's rows or over every clone: clone 0 can carry none of a + population that one or two households carry in the other clones. """ sources: dict[str, np.ndarray] = {} + clone_zero: dict[str, np.ndarray] = {} flags: list[dict[str, object]] = [] for entity, output, key in outputs: row: dict[str, object] = {"output": f"{entity}.{output}", "rate_key": key} if key == UK_UNDER_TWO_USAGE_SIGNAL_KEY: row["under_two_usage"] = True - measured = _measure_take_up_signal(frame, entity, output, key, contract) + if entity not in clone_zero: + clone_zero[entity] = geographic_clone_zero_rows(frame, entity) + measured = _measure_take_up_signal( + frame, entity, output, key, contract, clone_zero[entity] + ) if isinstance(measured, str): row["unmeasured"] = measured flags.append(row) @@ -840,10 +907,19 @@ def uk_take_up_signal_receipt( sources[entity] = record_source_households( frame, entity, stage="The take-up signal receipt", before_copies=True ) - support = distinct_count(sources[entity], measured.population, measured.weights) + support = distinct_count( + sources[entity], + measured.population & clone_zero[entity], + measured.weights, + ) + carriers = distinct_count( + sources[entity], measured.population, measured.weights + ) + # The figures are measured over every clone: withheld on either count. + withheld_on = carriers if 0 < carriers < UK_DISCLOSURE_FLOOR else support row["population_support"] = support_band(support) - row["target"] = small_cell_share(measured.target, support) - row["weighted_share"] = small_cell_share(measured.share, support) + row["target"] = small_cell_share(measured.target, withheld_on) + row["weighted_share"] = small_cell_share(measured.share, withheld_on) row["distinct_values"] = measured.unique_count if measured.nested_violations is not None: row["extended_without_universal_units"] = extreme_count( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py index 291ca86fcc..13a651e0ad 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py @@ -67,20 +67,23 @@ def uk_spine_transferred_only_code_hash() -> str: calendar-year weights, and the vendored-fact reader with the chronicle feed and its schema epochs); the annual income spells check (``frs_annual_income``); the apprentice draw's take-up contract - (``take_up_contract``); and the latent-attribute checks' binomial band - (``stochastic_assignment``). Its + (``take_up_contract``); the latent-attribute checks' binomial band + (``stochastic_assignment``); and the band donor check's copy-id fields, + through which it reads a capped seating's seats from the frame + (``copy_id_table``; microcosm#931). Its enum gate also checks the CGT asset type domain (``cgt_asset_type``). The assembled gate's own key, on which every later spine stage's chains, therefore carries none of them (the re-review of microcosm#1420, E; microcosm#931 ruling 6). An edit to one still re-keys the assembled gate wherever a stage before it hashes the module. Stages at or before - ``frs_brma``, the assembled phase's last, hash six of them: - ``stochastic_assignment`` and ``take_up_contract``; ``frs_annual_income``, - the module of ``frs_annual_income_spells``; and the vendored-fact reader, - the chronicle feed and its schema epochs, which ``frs_hmrc_spine_leaves`` - binds. Those two stages run before ``frs_brma`` on the clone-first spine - (microcosm#931). So an edit to any of the six re-keys every spine node + ``frs_brma``, the assembled phase's last, hash seven of them: + ``copy_id_table``, which the root stage binds; ``stochastic_assignment`` + and ``take_up_contract``; ``frs_annual_income``, the module of + ``frs_annual_income_spells``; and the vendored-fact reader, the chronicle + feed and its schema epochs, which ``frs_hmrc_spine_leaves`` binds. Those + two stages run before ``frs_brma`` on the clone-first spine + (microcosm#931). So an edit to any of the seven re-keys every spine node from the first stage that hashes it onwards, the assembled gate included. No stage before that gate hashes the other eleven, so an edit to one of those re-keys the stages that hash it, those after them and @@ -95,6 +98,7 @@ def uk_spine_transferred_only_code_hash() -> str: bus_use_incidence, cgt_asset_type, chronicle_feed, + copy_id_table, donor_uprating, energy_pricing, etb_services, @@ -124,6 +128,7 @@ def uk_spine_transferred_only_code_hash() -> str: chronicle_epoch, ledger_fact_vendoring, shared_ledger_targets, + copy_id_table, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py index ce49f61514..89afe885f0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py @@ -8,20 +8,24 @@ (``household_lineage.UK_COPY_LINEAGE``), and :func:`geographic_clone_count` reads ``K`` from it. The spine's ``geographic_expand`` stage writes the indices and every household's source columns -(:data:`UK_GEOGRAPHIC_SOURCE_COLUMNS`), and its ``geographic_identity`` -stage each clone's geography key (:data:`GEOGRAPHY_HOUSEHOLD_KEY_COLUMN`), -which the dense line's ``atomic_area_support`` names ``IDENTITY_COLUMN``. +(:data:`UK_GEOGRAPHIC_SOURCE_COLUMNS`) and mints the clones' ids in the +copy-id table's field :data:`GEOGRAPHIC_COPY_ID_FIELD`, and its +``geographic_identity`` stage each clone's geography key +(:data:`GEOGRAPHY_HOUSEHOLD_KEY_COLUMN`), which the dense line's +``atomic_area_support`` names ``IDENTITY_COLUMN``. The two index names lived in ``rowwise_dataset`` and the count in ``cgt_support``, which re-export them. This module holds the names and the count alone and imports nothing first-party, as ``annuity_trust_column`` holds its column's name. The stages that count by lineage read the household's name through ``household_lineage``, and their nodes hash this -module with the counting modules (``graph_kernels._COUNTING_MODULES``); the -CGT support split, which reads ``K`` through the count, hashes it too. The -full build's population and export kernels, which read the names, hash it as -well. So an edit here re-keys all of them, without binding the dense line's -``rowwise_dataset`` or ``atomic_area_support`` into the spine (microcosm#931). +module with the counting modules (``graph_kernels._COUNTING_MODULES``). The +CGT support split, the support-clip stages and property components read +``K`` through the count, and the WAS stages, which count nothing by lineage, +hash this module as a helper. The full build's population and export +kernels, which read the names, hash it as well. So an edit here re-keys all +of them, without binding the dense line's ``rowwise_dataset`` or +``atomic_area_support`` into the spine (microcosm#931). """ from __future__ import annotations @@ -31,6 +35,7 @@ __all__ = [ "ARTIFACT_CLONE_INDEX_COLUMN", + "GEOGRAPHIC_COPY_ID_FIELD", "GEOGRAPHY_HOUSEHOLD_KEY_COLUMN", "SOURCE_HOUSEHOLD_KEY_COLUMN", "SOURCE_YEAR_COLUMN", @@ -58,6 +63,11 @@ SOURCE_YEAR_COLUMN, SOURCE_HOUSEHOLD_KEY_COLUMN, ) +#: The copy-id table's field the clones' ids are minted in: clone ``k`` of a +#: row takes the row's id at ordinal ``k`` there, and every later copy keeps +#: its clone's ordinal (``copy_id_table``; spelled here so that this module +#: imports nothing first-party). +GEOGRAPHIC_COPY_ID_FIELD = "geographic_support" #: Each clone's geography key, which ``geographic_identity`` writes on every #: household of the spine and the dense line's ``uk.full.identity`` on every #: household of its pool (``atomic_area_support.IDENTITY_COLUMN`` is this @@ -76,7 +86,7 @@ def ladder_clone_index_column(entity: str) -> str: return f"{entity}_{ARTIFACT_CLONE_INDEX_COLUMN}" -def geographic_clone_count(household: pd.DataFrame) -> int: +def geographic_clone_count(household: pd.DataFrame, *, stage: str | None = None) -> int: """The frame's geographic clone count ``K``: one more than its largest index. The clone-first spine's ``geographic_expand`` stage (microcosm#931) gives @@ -84,39 +94,37 @@ def geographic_clone_count(household: pd.DataFrame) -> int: ``household_clone_index``, 0 to ``K - 1``; a household table without that column is one clone. Refuses a non-integer index column, a household without an index, a negative index and indices that are not exactly 0 to - ``K - 1`` (an index below the largest that no household carries). Each - refusal gives a count or the column's dtype, never an id or a value. + ``K - 1`` (an index below the largest that no household carries), since + a count above the clones the frame holds would widen whatever the caller + scales by it. Each refusal is a fixed sentence naming the column, the + calling ``stage`` where one is given and, for a column that is not + integer, its dtype: never a count, an id or a value. """ column = ladder_clone_index_column("household") if column not in household.columns: return 1 values = household[column] + rule = ( + f"{stage or 'A spine'} counts K geographic clones as the {column!r} " + "values 0 to K - 1 (microcosm#931)." + ) if pd.api.types.is_bool_dtype(values.dtype) or not ( pd.api.types.is_integer_dtype(values.dtype) ): raise ValueError( - "The CGT support split counts geographic clones on an integer " - f"{column!r} column; it has dtype {values.dtype}." + f"{column!r} has dtype {values.dtype}, not an integer one. {rule}" ) - rule = ( - f"the CGT support split counts K geographic clones as the {column!r} " - "values 0 to K - 1 (microcosm#931)." - ) - unindexed = int(values.isna().sum()) - if unindexed: - raise ValueError(f"{unindexed} household(s) carry no {column!r}; {rule}") + if bool(values.isna().any()): + raise ValueError(f"A household carries no {column!r}. {rule}") indices = values.to_numpy(dtype=np.int64) if not indices.size: return 1 - negative = int((indices < 0).sum()) - if negative: - raise ValueError(f"{negative} household(s) carry a negative {column!r}; {rule}") + if bool((indices < 0).any()): + raise ValueError(f"A household carries a negative {column!r}. {rule}") clone_count = int(indices.max()) + 1 - absent = clone_count - int(np.unique(indices).size) - if absent: + if int(np.unique(indices).size) != clone_count: raise ValueError( - f"{absent} of the {clone_count} indices from 0 to the largest are " - f"carried by no household; {rule}" + f"An index from 0 to the largest is carried by no household. {rule}" ) return clone_count diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py index 867123b5f5..857e6eacf4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py @@ -73,6 +73,7 @@ from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHIC_COPY_ID_FIELD, SOURCE_HOUSEHOLD_KEY_COLUMN, SOURCE_YEAR_COLUMN, UK_GEOGRAPHIC_SOURCE_COLUMNS, @@ -116,8 +117,9 @@ ] GEOGRAPHIC_EXPAND_STAGE_NAME = "geographic_expand" -#: The copy-id table's field the clones' ids are minted in. -GEOGRAPHIC_COPY_ID_FIELD = "geographic_support" +# The copy-id table's field the clones' ids are minted in +# (GEOGRAPHIC_COPY_ID_FIELD) is named in the geographic_clones leaf, which +# regional_home_values reads it from, and re-exported here. #: The most clones the field numbers: clone ordinals run below it. Exported #: so the spine and its graph can read the limit from here. GEOGRAPHIC_CLONE_LIMIT = copy_id_table.ordinal_bound(GEOGRAPHIC_COPY_ID_FIELD) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index faa19d4f5e..21a1aafe5f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -519,13 +519,16 @@ # households, count family roles, group tenure by the shared tenure map and # name the HBAI income predictor. The WAS stage, and the LISA, bus, LCFS, # ETB, landlord, reporter and housing stages below, fit through the QRF - # modules (microcosm#931). + # modules (microcosm#931). The two WAS stages count no support by lineage, + # so they bind the geographic clones' leaf for the clone count their + # support-clip receipts read (microcosm#931). "was_wealth": ( *_ENGINE_STAGE_MODULES, *_PINNED_TAB_MODULES, *_QRF_MODULES, frs_brma, gates, + geographic_clones, household_income, qrf_gates, support_clip, @@ -545,6 +548,7 @@ *_QRF_MODULES, frs_brma, gates, + geographic_clones, household_income, support_clip, tenure_constants, @@ -571,8 +575,10 @@ # The regional values rank as the dense node does and read its knots # (microcosm#1112), over the tenure map's owners; their keyed uniforms are # microcosm.graph's, which encodes each draw's coordinates as canonical - # JSON (microcosm#931). + # JSON, and the cap count reads each row's geographic clone from its id + # through the copy-id table (microcosm#931). "regional_home_values": ( + copy_id_table, property_localisation, rowwise_geography, tenure_constants, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py index ff6297a668..1b8ca6576d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py @@ -8,7 +8,9 @@ infers a branch from ids, weights or values. The UK's copy lineage is declared here (:data:`UK_COPY_LINEAGE`) for the shared readers in ``microcosm.build.source_lineage``: the receipts' true sources and the -FRS-trained fits' collapse of geographic clones read it too. +FRS-trained fits' collapse of geographic clones read it too, and +:func:`geographic_fold_receipt` records how a fit's fold met its survey +records. """ from __future__ import annotations @@ -26,10 +28,14 @@ fold_replicas, replica_keys, ) -from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.geographic_clones import ( + geographic_clone_count, + ladder_clone_index_column, +) from microcosm.build.uk_runtime.national_frame import UK_NATIONAL_SCHEMA __all__ = [ + "GEOGRAPHIC_FOLD_EVIDENCE", "GEOGRAPHIC_SUPPORT_BRANCH", "HIGH_VALUE_SUPPORT_COPY_COLUMN", "HIGH_VALUE_SUPPORT_INDEX_COLUMN", @@ -37,6 +43,7 @@ "UK_HOUSEHOLD_LINEAGE_COLUMNS", "collapse_geographic_clones", "fold_geographic_clones", + "geographic_fold_receipt", "uk_household_clone_paths", ] @@ -67,6 +74,10 @@ HIGH_VALUE_SUPPORT_INDEX_COLUMN, ) GEOGRAPHIC_SUPPORT_BRANCH = "geographic_support" +#: The evidence key under which a stage whose fit folds geographic clones +#: records the fold (:func:`geographic_fold_receipt`), on a frame of more +#: than one clone only. +GEOGRAPHIC_FOLD_EVIDENCE = "geographic_fold" _BASE_SUPPORT_CHANNEL = "frs" _SPI_SUPPORT_CHANNEL = "spi" #: The UK's copy lineage for the shared readers in @@ -222,6 +233,71 @@ def collapse_geographic_clones( return fold.kept, fold.weights +def geographic_fold_receipt( + tables: Mapping[str, pd.DataFrame], + entity: str, + fold: ReplicaFold, + *, + stage: str, +) -> dict[str, object] | None: + """How a fit's geographic fold met its survey records, or None at one clone. + + ``fold`` is :func:`fold_geographic_clones`' fold of ``entity``'s rows + over ``tables``. On a frame of more than one geographic clone + (:func:`~microcosm.build.uk_runtime.geographic_clones.geographic_clone_count`) + the receipt holds: + + - ``clone_count``: the frame's clone count ``K``; + - ``records``: the true source households of the records the fit + selected; + - ``varied_records``: the source households of the selected records + whose clones did not all fold into one row, since they differ on what + the fit reads; + - ``varied_share``: the share of the selected source households that + vary, floored to the hundredth, where both the households that vary + and those that do not read as rounded numbers rather than bands; + otherwise it repeats ``varied_records``. A share to the hundredth of a + few dozen sources would give back where in its band either count + lies. + + Both counts are of distinct true sources on rows of positive weight, + rounded as every receipt count is (``receipt_counts.rounded_count``), so + a household's clones and copies count once. A frame of one clone returns + None and its evidence is left as it was. Variety is recorded, never + refused: clones that differ stay rows of their own in the fit. + """ + + clone_count = geographic_clone_count(tables["household"], stage=stage) + if clone_count == 1: + return None + # receipt_counts reads this module's lineage, so it is imported here. + from microcosm.build.uk_runtime import receipt_counts + + sources = receipt_counts.record_source_households(tables, entity, stage=stage) + selected = fold.representative >= 0 + records = receipt_counts.distinct_count(sources, selected, fold.weights) + varied = receipt_counts.distinct_count(sources, fold.varied, fold.weights) + varied_band = receipt_counts.rounded_count(varied) + # A varied record is a selected one, so the rest are the selected sources + # none of whose records varied. + steady_band = receipt_counts.rounded_count(records - varied) + share_given = ( + varied > 0 + and not isinstance(varied_band, str) + and not isinstance(steady_band, str) + ) + return { + "clone_count": clone_count, + "records": receipt_counts.rounded_count(records), + "varied_records": varied_band, + "varied_share": ( + receipt_counts.floored_share(varied / records) + if share_given + else varied_band + ), + } + + def without_geographic_branch( paths: Sequence[tuple[tuple[str, int], ...]], ) -> list[tuple[tuple[str, int], ...]]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 8cfa63163a..8474840e2a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -67,6 +67,7 @@ gambling_participation_operation, top_up_gambling_participation, ) +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_GROSS_INCOME_DEDUCTIONS, UK_HBAI_NET_INCOME_PREDICTOR, @@ -1110,7 +1111,12 @@ def __call__(self, frame: Frame) -> Frame: gate_calibration=_qrf_gate_calibration(self.stage), ) clip_result = support_clip_to_donor( - imputation.draws, donor, exempt=support_clip_exempt(self.stage) + imputation.draws, + donor, + exempt=support_clip_exempt(self.stage), + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_LCFS_CONSUMPTION_STAGE_NAME + ), ) household_draws, shares_outside = clip_excise_shares( clip_result.clipped, shares @@ -5517,6 +5523,7 @@ def support_clip_to_donor( donor: pd.DataFrame, *, exempt: set[str] | None = None, + clone_count: int = 1, ) -> UKSupportClipResult: return support_clip_to_donor_with_receipt( draws, @@ -5524,6 +5531,7 @@ def support_clip_to_donor( columns=UK_LCFS_CONSUMPTION_TARGET_COLUMNS, stage=UK_LCFS_CONSUMPTION_STAGE_NAME, exempt=exempt, + clone_count=clone_count, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py index 598d9ef2d2..47ed196940 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py @@ -47,6 +47,7 @@ nts_band_shares, ) from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_GROSS_INCOME_VARIABLES, hbai_gross_income, @@ -2016,6 +2017,9 @@ def __call__(self, frame: Frame) -> Frame: donor_support, columns=UK_NTS_SUPPORT_CLIP_COLUMNS, stage=UK_NTS_BUS_TRAVEL_STAGE_NAME, + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_NTS_BUS_TRAVEL_STAGE_NAME + ), ) clipped = clip.clipped # Outputs. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py index 55910324e9..ce9f33dc20 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py @@ -75,6 +75,7 @@ keyed_tie_order, stable_identity_uniforms, ) +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.hmrc_property_rental import ( HMRC_PROPERTY_RENTAL_RECEIPTS_BAND_LOWER_BOUNDS, HMRC_PROPERTY_RENTAL_RECORD_SETS, @@ -915,9 +916,13 @@ class UKPropertyComponentsResult: channels: Mapping[str, Any] residuals: Mapping[str, Any] coherence: Mapping[str, Any] + #: The frame's geographic clone count ``K`` (microcosm#931). The receipt + #: gives it only when it is more than one, and the stage's check then + #: allows ``K`` times its declared row maxima. + clone_count: int = 1 def evidence(self) -> dict[str, object]: - return { + evidence: dict[str, object] = { "stage": UK_PROPERTY_COMPONENTS_STAGE_NAME, "facts": dict(self.facts), "donor": dict(self.donor), @@ -927,6 +932,9 @@ def evidence(self) -> dict[str, object]: "residuals": dict(self.residuals), "coherence": dict(self.coherence), } + if self.clone_count > 1: + evidence["clone_count"] = self.clone_count + return evidence @dataclass @@ -994,6 +1002,9 @@ def __call__(self, frame: Frame) -> Frame: person = frame.table("person").copy() household = frame.table("household") + clone_count = geographic_clone_count( + household, stage=UK_PROPERTY_COMPONENTS_STAGE_NAME + ) missing = sorted(PROPERTY_COMPONENTS_PERSON_INPUTS - set(person.columns)) if missing: raise PropertyComponentsError(f"the spine's person table lacks {missing}.") @@ -1167,6 +1178,7 @@ def __call__(self, frame: Frame) -> Frame: channels=channels, residuals=residuals, coherence=coherence, + clone_count=clone_count, ) return result diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_home_values.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_home_values.py index c1916410a5..550d9ada22 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_home_values.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_home_values.py @@ -22,6 +22,15 @@ The value it replaces is kept in ``main_residence_value_rank_basis``, which the dense node ranks on, so the dense line's values, wealth and bands do not change; the release boundary drops the column on both lines. + +The receipt's ``sources_at_cap`` counts the source households at the value cap +in geographic clone 0's rows (microcosm#931). Each clone ranks on its own +draw, so a union over the clones would grow with the clone count, and the +stage's check holds the count to one declared maximum. A row's clone is its +id's ordinal in the copy-id table's ``geographic_support`` field, which the +clones' ids are minted in and every later copy keeps (clone 0's ordinal is 0), +so the stage reads it without the clone index. A frame that carries the index +as well is refused where the two disagree, as on clones minted another way. """ from __future__ import annotations @@ -33,7 +42,12 @@ import numpy as np from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHIC_COPY_ID_FIELD, + ladder_clone_index_column, +) from microcosm.build.uk_runtime.household_lineage import HIGH_VALUE_SUPPORT_COPY_COLUMN from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, @@ -221,7 +235,24 @@ def assign_regional_home_values( mass_log=(*frame.mass_log, mass), ) validate_uk_national_frame(result) - at_cap = distinct_count(source_ids, target & (new_value >= parsed.cap), weights) + # Counted in geographic clone 0's rows, so the count does not grow with + # the clone count (microcosm#931); at one clone every row is clone 0's. + clone = copy_id_table.ordinal( + household["household_id"].to_numpy(), GEOGRAPHIC_COPY_ID_FIELD + ) + index_column = ladder_clone_index_column("household") + if index_column in household.columns and not np.array_equal( + household[index_column].to_numpy(), clone + ): + raise ValueError( + "The regional home values stage reads each household's geographic " + "clone from its id, which disagrees with the frame's " + f"{index_column!r} (microcosm#931)." + ) + clone_zero = clone == 0 + at_cap = distinct_count( + source_ids, target & (new_value >= parsed.cap) & clone_zero, weights + ) receipt = { "knots_basis_year": parsed.basis_year, "knots_inputs_sha256": parsed.inputs_sha256, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py index 9ea0bfd893..c9c9149dba 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/salary_sacrifice.py @@ -31,7 +31,11 @@ ) from microcosm.build.uk_runtime.cgt_support import HOUSEHOLD_IS_CGT_SUPPORT_COPY from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column -from microcosm.build.uk_runtime.household_lineage import collapse_geographic_clones +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_FOLD_EVIDENCE, + fold_geographic_clones, + geographic_fold_receipt, +) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, @@ -498,6 +502,9 @@ class UKSalarySacrificeResult: bonus_receipt: Mapping[str, object] | None = None #: The fit's gate outcome, when the gate is not in-sample. gate_outcomes: Mapping[str, str] | None = None + #: How the forest's geographic fold met its survey records + #: (``household_lineage.geographic_fold_receipt``); None on one clone. + geographic_fold: Mapping[str, object] | None = None def evidence(self) -> dict[str, object]: return { @@ -551,6 +558,11 @@ def evidence(self) -> dict[str, object]: if self.gate_outcomes else {} ), + **( + {} + if self.geographic_fold is None + else {GEOGRAPHIC_FOLD_EVIDENCE: dict(self.geographic_fold)} + ), } @@ -665,17 +677,23 @@ def impute_salary_sacrifice( person_weights = person_households.map(household_weights).to_numpy(dtype=float) # The forest trains on each asked respondent once: their geographic # clones that agree on the predictors and the sacrifice fold into their - # first row with their weight, clones that differ stay rows of their own, - # and every other copy stays its own row; a frame without geographic - # clones keeps every row and weight - # (household_lineage.collapse_geographic_clones, microcosm#931). - kept, fit_weights = collapse_geographic_clones( - {"person": person, "household": household}, + # lowest-id clone with their weight, clones that differ stay rows of + # their own, and every other copy stays its own row; a frame without + # geographic clones keeps every row and weight + # (household_lineage.fold_geographic_clones, microcosm#931). On more than + # one clone the evidence records the fold. + lineage_tables = {"person": person, "household": household} + fold = fold_geographic_clones( + lineage_tables, "person", training_mask.to_numpy(dtype=bool), person_weights, inputs=numeric[[*SALSAC_PREDICTORS, SALSAC_OUTPUT]].reset_index(drop=True), ) + geographic_fold = geographic_fold_receipt( + lineage_tables, "person", fold, stage="The salary-sacrifice stage" + ) + kept, fit_weights = fold.kept, fold.weights training_mask = pd.Series(kept, index=training_mask.index) training = numeric.loc[training_mask, [*SALSAC_PREDICTORS, SALSAC_OUTPUT]].copy() training["_fit_weight"] = fit_weights[kept] @@ -938,6 +956,7 @@ def bonus_sources(mask: np.ndarray) -> int: if SALSAC_QRF_GATE_CALIBRATION != UK_DEFAULT_GATE_CALIBRATION else None ), + geographic_fold=geographic_fold, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py index d7d068e3dd..cf705aff25 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py @@ -212,38 +212,74 @@ #: share's cap bound (UKDS disclosure control). SPI_INCOME_BAND_FULL_SEATING = "full" SPI_INCOME_BAND_CAPPED_SEATING = "capped" +#: The relative tolerance ``funding_within_share`` holds every funding factor +#: to: the stage gate's float tolerance, eight units in the last place +#: (``stage_health._FLOAT_RELATIVE_TOLERANCE``; a test holds the two equal). +SPI_INCOME_BAND_FUNDING_RELATIVE_TOLERANCE = 8.0 * float(np.finfo(np.float64).eps) +#: How far, relative to the reallocated mass, ``funding_matches_bands`` lets +#: the funding strata's and the bands' donor mass each sit from it: the stage +#: gate's declared ``maximum_relative_mass_deviation`` (a test holds the two +#: equal), or the float tolerance where the declared deviation is finer than +#: it. +SPI_INCOME_BAND_FUNDING_MASS_TOLERANCE = max( + 1e-9, SPI_INCOME_BAND_FUNDING_RELATIVE_TOLERANCE +) +#: Each band's donors and carriers in distinct source households: a full +#: seating's receipt leaves them out, and a capped seating's gives them, +#: rounded, as ``donor_households`` and ``carriers``. +_BAND_SOURCE_COUNT_KEYS = ("donor_source_households", "carrier_source_households") @dataclass(frozen=True) class UKSPIIncomeBandDonorResult: """Frame with the reserved band rows and the receipt the gate reads. - ``seating_scale`` is the exact scale the seats were cut at; the receipt - records it as full or capped. These counts are already in their receipt - forms, counted in distinct source households and rounded (UKDS disclosure - control): ``eligible_households``, each band row's ``eligible_households`` - (clone 0's, which a one-clone spine counts alike), the funding rows' - ``donor_households`` and ``carrier_region_counts``. A band row's + The result keeps the stage's exact figures, and :meth:`evidence` writes + the receipt from them (microcosm#931). The exact figures stay out of the + repr, and the receipt gives each in its seating's form: ``seating_scale`` + as full or capped; the band rows' counts in source households + (``donor_source_households``, ``carrier_source_households``) only through + the capped form; each funding row's incumbent mass, donor mass and factor + never. The repr leaves out the frame as well. ``expected_donor_count`` is + the plan's. + + Every receipt counts these in distinct source households and rounds them + (``receipt_counts``): ``eligible_households``, each band row's + ``eligible_households`` (clone 0's, which a one-clone spine counts + alike), the funding rows' ``donor_households`` and + ``carrier_region_counts``. A full seating seats every band's plan, so the + receipt gives the band rows' ``donor_households`` and ``carriers``, + ``donor_count`` and ``reallocated_mass`` exact, and a band row's ``carrier_mean_age`` is withheld below three carrier source households or - three seats per clone. The band rows' ``donor_households`` and - ``carriers``, and ``donor_count``, stay exact row counts, which the stage - gate reads exactly; a band row's ``carrier_regions`` is an exact count of - regions that no check reads. ``expected_donor_count`` is the plan's. + three seats per clone. A capped seating's receipt gives each band's donors + and carriers, and ``donor_count``, in rounded distinct source households, + withholds each band's ``weighted_taxpayers`` and ``carrier_mean_age`` and + the ``reallocated_mass``, and leaves each band's ``carrier_regions`` out. + At every seating the funding rows keep each stratum and its rounded + donors and withhold its masses and factor, and two booleans give the + funding: ``funding_within_share``, every factor at least one less the + declared reallocation share, to the gate's relative tolerance + (:data:`SPI_INCOME_BAND_FUNDING_RELATIVE_TOLERANCE`); and + ``funding_matches_bands``, the funding strata and the bands each carrying + the reallocated mass (:data:`SPI_INCOME_BAND_FUNDING_MASS_TOLERANCE`). """ - frame: Frame - band_rows: tuple[Mapping[str, object], ...] - funding_rows: tuple[Mapping[str, object], ...] + frame: Frame = field(repr=False) + band_rows: tuple[Mapping[str, object], ...] = field(repr=False) + funding_rows: tuple[Mapping[str, object], ...] = field(repr=False) minimum_donors_per_band: int maximum_donor_weight: float maximum_donor_household_share: float maximum_reallocation_share: float - seating_scale: float + seating_scale: float = field(repr=False) mass_scale: float eligible_households: int | str expected_donor_count: int - donor_count: int - reallocated_mass: float + donor_count: int = field(repr=False) + donor_source_households: int = field(repr=False) + reallocated_mass: float = field(repr=False) + funding_within_share: bool + funding_matches_bands: bool old_total: float new_total: float seed: int @@ -255,6 +291,7 @@ class UKSPIIncomeBandDonorResult: funding_strata: tuple[str, ...] def evidence(self) -> dict[str, object]: + full = self.seating_scale >= 1.0 return { "stage": UK_SPI_INCOME_BAND_DONORS_STAGE_NAME, "minimum_donors_per_band": self.minimum_donors_per_band, @@ -264,28 +301,74 @@ def evidence(self) -> dict[str, object]: "clone_count": self.clone_count, "funding_strata": list(self.funding_strata), "seating_scale": ( - SPI_INCOME_BAND_FULL_SEATING - if self.seating_scale >= 1.0 - else SPI_INCOME_BAND_CAPPED_SEATING + SPI_INCOME_BAND_FULL_SEATING if full else SPI_INCOME_BAND_CAPPED_SEATING ), "mass_scale": self.mass_scale, "sample_fraction": self.sample_fraction, "eligible_households": self.eligible_households, "expected_donor_count": self.expected_donor_count, - "donor_count": self.donor_count, - "reallocated_mass": self.reallocated_mass, + "donor_count": ( + self.donor_count + if full + else rounded_count(self.donor_source_households) + ), + "reallocated_mass": self.reallocated_mass if full else UK_WITHHELD, + "funding_within_share": self.funding_within_share, + "funding_matches_bands": self.funding_matches_bands, "mass": {"old_total": self.old_total, "new_total": self.new_total}, "draw": {"seed": self.seed, "salt": SPI_INCOME_BAND_DONOR_DRAW_SALT}, "taxpayer_count_resource": SPI_INCOME_BAND_TAXPAYER_RESOURCE, "taxpayer_count_period": self.taxpayer_period, "propensity": SPI_INCOME_BAND_DONOR_PROPENSITY, "propensity_cells": self.propensity_cells, - "bands": [dict(row) for row in self.band_rows], - "funding": [dict(row) for row in self.funding_rows], + "bands": [_band_receipt(row, full=full) for row in self.band_rows], + "funding": [_funding_receipt(row) for row in self.funding_rows], "carrier_region_counts": dict(self.carrier_region_counts), } +def _band_receipt(row: Mapping[str, object], *, full: bool) -> dict[str, object]: + """A band row as the receipt gives it, at full or capped seating. + + At full seating, the row as the stage counted it, without its counts in + source households. At capped seating, its donors and carriers in rounded + distinct source households; its weighted taxpayers and its carriers' mean + age withheld; its carriers' regions left out. + """ + + if full: + return { + key: value + for key, value in row.items() + if key not in _BAND_SOURCE_COUNT_KEYS + } + return { + "lower_bound": row["lower_bound"], + "expected_donor_households": row["expected_donor_households"], + "donor_households": rounded_count(int(row["donor_source_households"])), + "carriers": rounded_count(int(row["carrier_source_households"])), + "eligible_households": row["eligible_households"], + "planned_donor_weight": row["planned_donor_weight"], + "donor_weight": row["donor_weight"], + "weighted_taxpayers": UK_WITHHELD, + "published_taxpayers": row["published_taxpayers"], + "carrier_mean_age": UK_WITHHELD, + } + + +def _funding_receipt(row: Mapping[str, object]) -> dict[str, object]: + """A funding row as the receipt gives it at every seating: the stratum and + its rounded donor households, its masses and factor withheld.""" + + return { + "stratum": row["stratum"], + "donor_households": row["donor_households"], + "incumbent_mass": UK_WITHHELD, + "donor_mass": UK_WITHHELD, + "factor": UK_WITHHELD, + } + + def spi_income_band_donor_plan( band_taxpayers: Mapping[int, float], *, @@ -693,19 +776,14 @@ def stack_spi_income_band_donors( int(pd.unique(candidate_households[positions[eligible[positions]]]).size) for positions in clone_positions ] - short = [count for count in found if count < seats[band]] - if short and clone_count == 1: + # The refusal names the band and, on a cloned spine, that a clone fell + # short; it gives no count (microcosm#931). + if any(count < seats[band] for count in found): raise ValueError( - f"SPI income band donors need {seats[band]} candidate households " - f"with positive weight and propensity for the band from {band}; " - f"found {found[0]}." - ) - if short: - raise ValueError( - f"SPI income band donors need {seats[band]} candidate households " - f"with positive weight and propensity for the band from {band} in " - f"each of the {clone_count} geographic clones; {len(short)} " - f"clone(s) found fewer, the fewest {min(short)}." + "SPI income band donors found fewer candidate households with " + "positive weight and propensity for the band from " + f"{band} than the band seats" + + ("." if clone_count == 1 else " in a geographic clone.") ) # The receipt's count for the band is clone 0's, in distinct source # households: clone 0 walks exactly what one clone walks, so the count @@ -751,8 +829,8 @@ def stack_spi_income_band_donors( break if len(taken) != seats[band]: raise ValueError( - f"SPI income band donors could not seat {seats[band]} " - f"households for the band from {band}" + "SPI income band donors could not seat every seat of the band " + f"from {band}" + ("." if clone_count == 1 else " in a geographic clone.") ) @@ -880,8 +958,9 @@ def stack_spi_income_band_donors( # survey household (UKDS disclosure control, receipt_counts): the eligible # counts, the funding rows' donors and the carriers per region. A band's # mean age is withheld below three carrier source households or three - # seats per clone. The band rows' donor and carrier counts stay exact row - # counts, because the stage gate reads them exactly. + # seats per clone. The band rows keep their exact row counts and their + # counts in source households beside them, and the receipt gives each in + # the form its seating takes (UKSPIIncomeBandDonorResult.evidence). donor_sources = source_household_ids( donor_household, stage=UK_SPI_INCOME_BAND_DONORS_STAGE_NAME ) @@ -910,6 +989,8 @@ def stack_spi_income_band_donors( "expected_donor_households": int(plan[band][0]) * clone_count, "donor_households": int(mask.sum()), "carriers": int(carriers.sum()), + "donor_source_households": distinct_count(donor_sources, mask), + "carrier_source_households": carrier_support, "eligible_households": rounded_count(band_eligible[band]), "planned_donor_weight": float(clone_weight[band]), "donor_weight": float(clone_weight[band] * mass_scale), @@ -937,6 +1018,7 @@ def stack_spi_income_band_donors( } for stratum in sorted(incumbent_mass.index, key=str) ] + reallocated_mass = float(donor_weights.sum()) carrier_region_labels = carrier_region.to_numpy() return UKSPIIncomeBandDonorResult( frame=result, @@ -952,7 +1034,16 @@ def stack_spi_income_band_donors( expected_donor_count=int(sum(count for count, _ in plan.values())) * clone_count, donor_count=len(selected_household), - reallocated_mass=float(donor_weights.sum()), + donor_source_households=distinct_count( + donor_sources, np.ones(len(donor_sources), dtype=bool) + ), + reallocated_mass=reallocated_mass, + funding_within_share=_funding_within_share( + funding_rows, maximum_reallocation_share=maximum_reallocation_share + ), + funding_matches_bands=_funding_matches_bands( + band_rows, funding_rows, reallocated_mass=reallocated_mass + ), old_total=old_total, new_total=new_total, seed=seed, @@ -973,6 +1064,53 @@ def stack_spi_income_band_donors( ) +def _funding_within_share( + funding_rows: Sequence[Mapping[str, object]], + *, + maximum_reallocation_share: float, +) -> bool: + """Whether every funding stratum kept at least one less the declared share. + + Each stratum's factor, the share of its incumbents' mass they keep, must + lie in ``[1 - share, 1]``, the lower end to the stage gate's relative + tolerance (:data:`SPI_INCOME_BAND_FUNDING_RELATIVE_TOLERANCE`). The gate + reads the boolean, and holds ``1 - share`` to its declared minimum funding + factor (microcosm#931). + """ + + floor = 1.0 - float(maximum_reallocation_share) + return all( + floor * (1.0 - SPI_INCOME_BAND_FUNDING_RELATIVE_TOLERANCE) + <= float(row["factor"]) + <= 1.0 + for row in funding_rows + ) + + +def _funding_matches_bands( + band_rows: Sequence[Mapping[str, object]], + funding_rows: Sequence[Mapping[str, object]], + *, + reallocated_mass: float, +) -> bool: + """Whether the bands and the funding strata each carry the reallocated mass. + + The bands carry each band's donor weight times its donors, the funding + strata their donor mass; each sum must lie within + :data:`SPI_INCOME_BAND_FUNDING_MASS_TOLERANCE` of the reallocated mass, + relative to it, as the stage gate held them (microcosm#931). + """ + + banded = 0.0 + for row in band_rows: + banded += float(row["donor_weight"]) * int(row["donor_households"]) + funded = 0.0 + for row in funding_rows: + funded += float(row["donor_mass"]) + allowed = SPI_INCOME_BAND_FUNDING_MASS_TOLERANCE * max(abs(reallocated_mass), 1.0) + return all(abs(value - reallocated_mass) <= allowed for value in (banded, funded)) + + def _funding_strata(household: pd.DataFrame) -> pd.Series: """Each household's funding stratum label. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_employment_shell.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_employment_shell.py index e2dd08f659..a62700f80a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_employment_shell.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_employment_shell.py @@ -38,7 +38,11 @@ integer_record_keys, stable_identity_uniforms, ) -from microcosm.build.uk_runtime.household_lineage import collapse_geographic_clones +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_FOLD_EVIDENCE, + fold_geographic_clones, + geographic_fold_receipt, +) from microcosm.build.uk_runtime.occupation_constants import ( UK_OCCUPATION_STATUS_GROUP, UK_SOC2020_NO_OCCUPATION, @@ -555,12 +559,14 @@ def impute_spi_employment_shell( raise ValueError(f"{stage} needs FRS adults with an occupation to train on.") # The fit and the donors see each survey record once: its geographic # clones that agree on the predictors, the occupation and the industry it - # donates fold into its first row with their weight, clones that differ - # stay rows of their own, and every other copy stays its own row; a frame - # without geographic clones keeps every row and weight - # (household_lineage.collapse_geographic_clones, microcosm#931). - training, fit_weights = collapse_geographic_clones( - {"person": person, "household": household}, + # donates fold into its lowest-id clone with their weight, clones that + # differ stay rows of their own, and every other copy stays its own row; + # a frame without geographic clones keeps every row and weight + # (household_lineage.fold_geographic_clones, microcosm#931). On more than + # one clone the receipt records the fold. + lineage_tables = {"person": person, "household": household} + fold = fold_geographic_clones( + lineage_tables, "person", training, weights, @@ -574,6 +580,10 @@ def impute_spi_employment_shell( axis=1, ), ) + geographic_fold = geographic_fold_receipt( + lineage_tables, "person", fold, stage=stage + ) + training, fit_weights = fold.kept, fold.weights # The training rows in person_id order, so neither the fit (the early # stopping split, the folds) nor the donor ties depend on how the frame @@ -692,6 +702,8 @@ def impute_spi_employment_shell( "held_out": industry_held_out, }, } + if geographic_fold is not None: + receipt[GEOGRAPHIC_FOLD_EVIDENCE] = geographic_fold return person_out, receipt diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_housing_shell.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_housing_shell.py index 6135b5c1b1..01df8a6876 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_housing_shell.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_housing_shell.py @@ -38,7 +38,11 @@ UK_COUNCIL_TAX_BANDS_BY_REGION, UK_NATION_BY_REGION, ) -from microcosm.build.uk_runtime.household_lineage import collapse_geographic_clones +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_FOLD_EVIDENCE, + fold_geographic_clones, + geographic_fold_receipt, +) from microcosm.build.uk_runtime.qrf_gates import ( UK_DEFAULT_GATE_CALIBRATION, UK_GATE_OUTCOMES_EVIDENCE, @@ -543,12 +547,12 @@ def impute_spi_housing_shell( predictors = household_housing_predictors(person, household) targets = household_housing_targets(person, household) # The model trains on each survey household once: its geographic clones - # that agree on the predictors and targets fold into its first row with - # their weight, clones that differ stay rows of their own, and every + # that agree on the predictors and targets fold into its lowest-id clone + # with their weight, clones that differ stay rows of their own, and every # other copy stays its own row; a frame without geographic clones keeps - # every row and weight (household_lineage.collapse_geographic_clones, - # microcosm#931). - training, fit_weights = collapse_geographic_clones( + # every row and weight (household_lineage.fold_geographic_clones, + # microcosm#931). On more than one clone the receipt records the fold. + fold = fold_geographic_clones( {"household": household}, "household", (channel == FRS_CHANNEL) & (weights.to_numpy() > 0.0), @@ -560,7 +564,14 @@ def impute_spi_housing_shell( .reindex(household["household_id"].to_numpy()) .reset_index(drop=True), ) - fit_weights = pd.Series(fit_weights, index=household["household_id"]) + geographic_fold = geographic_fold_receipt( + {"household": household}, + "household", + fold, + stage=UK_SPI_HOUSING_SHELL_STAGE_NAME, + ) + training = fold.kept + fit_weights = pd.Series(fold.weights, index=household["household_id"]) training_ids = household.loc[training, "household_id"] recipient_ids = household.loc[channel == SPI_CHANNEL, "household_id"] if training_ids.empty or recipient_ids.empty: @@ -628,6 +639,8 @@ def impute_spi_housing_shell( receipt[UK_GATE_OUTCOMES_EVIDENCE] = gate_outcomes( UK_SPI_HOUSING_SHELL_STAGE_NAME, model.amount_model ) + if geographic_fold is not None: + receipt[GEOGRAPHIC_FOLD_EVIDENCE] = geographic_fold return person_out, household_out, receipt diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py index 011bf3eb56..5caa451606 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py @@ -50,7 +50,10 @@ LOSS_RELIEF_COLUMN, self_employment_income_after_loss_relief, ) -from microcosm.build.uk_runtime.household_lineage import collapse_geographic_clones +from microcosm.build.uk_runtime.household_lineage import ( + fold_geographic_clones, + geographic_fold_receipt, +) from microcosm.build.uk_runtime.pension_coherence import ( cap_net_pay_contributions, coherence_receipt, @@ -1176,6 +1179,9 @@ class UKSPIIncomeImputationResult: #: The predictor levels each stage-1 gate's calibration offsets on their #: own, for a stage 1 that declares ``gate_level_columns``. gate_levels: Mapping[str, object] | None = None + #: How the FRS forests' geographic fold met their survey records + #: (``household_lineage.geographic_fold_receipt``); None on one clone. + geographic_fold: Mapping[str, object] | None = None @dataclass(frozen=True) @@ -1532,20 +1538,24 @@ def impute_uk_spi_income_support( if not training_people.any(): raise ValueError("FRS-only stage has no canonical base training rows.") # Both forests train on each survey record once: its geographic clones - # that agree on what the forests read fold into its first row with their - # weight, clones that differ stay rows of their own, and every other - # copy stays its own row; a frame without geographic clones keeps every - # row and weight (household_lineage.collapse_geographic_clones, - # microcosm#931). - kept, training_weight_values = collapse_geographic_clones( - {"person": person, "household": household}, + # that agree on what the forests read fold into its lowest-id clone with + # their weight, clones that differ stay rows of their own, and every + # other copy stays its own row; a frame without geographic clones keeps + # every row and weight (household_lineage.fold_geographic_clones, + # microcosm#931). On more than one clone the evidence records the fold. + lineage_tables = {"person": person, "household": household} + fold = fold_geographic_clones( + lineage_tables, "person", training_people.to_numpy(dtype=bool), person_weights.to_numpy(dtype=np.float64), inputs=_forest_inputs(person, household), ) - training_people = pd.Series(kept, index=person.index) - training_weights = pd.Series(training_weight_values, index=person.index) + geographic_fold = geographic_fold_receipt( + lineage_tables, "person", fold, stage="The SPI income stage" + ) + training_people = pd.Series(fold.kept, index=person.index) + training_weights = pd.Series(fold.weights, index=person.index) # The reference forest's population mirrors the draw's: FRS claimants and # partners aged 16 and over with taxable income under the same rule. reference_people = ( @@ -1650,6 +1660,7 @@ def fit_reference(encoded: pd.DataFrame, observed: pd.DataFrame): "the person table (the support lineage column the manifest declares)." ) identity_column = SPI_TRANSPORT_KEY_COLUMN + recipient_identities = person.loc[spi_people, identity_column].to_numpy() transport = cross_fit_transport( fit_reference, training_predictors=reference_predictors, @@ -1661,12 +1672,21 @@ def fit_reference(encoded: pd.DataFrame, observed: pd.DataFrame): recipient_predictors=recipient_predictors, encoded_recipients=encoded_recipient, observed=observed_analogues(person.loc[spi_people]), - identities=person.loc[spi_people, identity_column].to_numpy(), + identities=recipient_identities, groups=recipient_predictors["main_source"].to_numpy(), seed=seed, folds=SPI_TRANSPORT_FOLDS, allow_in_sample=allow_in_sample_reference, outputs=SPI_INCOME_QRF_OUTPUT_COLUMNS, + # A copy's twin may have folded into its record's lowest-id clone + # above, so the twin check finds the twin where the training rows + # hold it, while each copy's uniforms stay keyed on its own twin + # (microcosm#931). + twin_identities=_folded_twin_identities( + person[identity_column].to_numpy(), + fold.representative, + recipient_identities, + ), ) # The tape forest reads a located sign uniform as the reference's gate # wrote it only when both gates order the same classes (microcosm#840): a @@ -2030,6 +2050,7 @@ def fit_reference(encoded: pd.DataFrame, observed: pd.DataFrame): if stage1_level_columns else None ), + geographic_fold=geographic_fold, ) @@ -3324,6 +3345,38 @@ def _forest_inputs(person: pd.DataFrame, household: pd.DataFrame) -> pd.DataFram ) +def _folded_twin_identities( + identities: np.ndarray, representative: np.ndarray, recipients: np.ndarray +) -> np.ndarray: + """Each recipient's twin as the folded training rows hold it. + + ``identities`` is every person's transport key, ``representative`` the + stage's geographic fold's map of each training row to the row it trains + on (-1 off the training rows), and ``recipients`` the recipients' keys, + each its own twin's. A twin folded into another clone's row is held by + that row, so its key reads as that row's; every other key is returned as + it is. Without geographic clones no training row folds into another, so + the recipients' keys come back unchanged (microcosm#931). + """ + + rows = np.flatnonzero(representative >= 0) + folded = rows[representative[rows] != rows] + recipients = np.asarray(recipients) + if not folded.size: + return recipients + held = pd.Index(identities[folded]) + if not held.is_unique: + raise ValueError( + "The SPI income forests' folded training rows repeat a transport " + "key, so a copy's twin would be ambiguous." + ) + position = held.get_indexer(recipients) + twins = recipients.copy() + found = position >= 0 + twins[found] = identities[representative[folded[position[found]]]] + return twins + + def _person_predictors( person: pd.DataFrame, household: pd.DataFrame, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_rank_transport.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_rank_transport.py index 596ec88f39..cb12ab09e9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_rank_transport.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_rank_transport.py @@ -285,6 +285,7 @@ def cross_fit_transport( folds: int, allow_in_sample: bool = False, outputs: Sequence[str] = SPI_INCOME_QRF_OUTPUT_COLUMNS, + twin_identities: np.ndarray | None = None, ) -> SPITransportResult: """Locate every recipient on a reference forest that never saw its cell. @@ -295,6 +296,14 @@ def cross_fit_transport( forest that locates a copy was fitted without its twin. Each fold's recipients are located by :func:`transport_uniforms`; the receipt is assembled over every recipient and names the folds. + + ``identities`` keys every recipient's uniforms. ``twin_identities``, one + per recipient and ``identities`` by default, names the training row that + holds each recipient's twin, and is read only to check that a recipient + takes its twin's fold. The two differ where the training rows fold a + twin's geographic clones into another clone's row: the twin is then + held by that row, while the copy keeps the uniforms keyed on its own + twin (microcosm#931). """ assert_analogue_map_covers(outputs) @@ -302,6 +311,13 @@ def cross_fit_transport( identities = np.asarray(identities) if identities.shape != (n,): raise ValueError("SPI transport identities must have one entry per recipient.") + twin_identities = ( + identities if twin_identities is None else np.asarray(twin_identities) + ) + if twin_identities.shape != (n,): + raise ValueError( + "SPI transport twin identities must have one entry per recipient." + ) if len(observed) != n or not observed.index.equals(encoded_recipients.index): raise ValueError("SPI transport observed analogues must align to recipients.") if len(training_folds) != len(encoded_training): @@ -317,14 +333,15 @@ def cross_fit_transport( # over the same cells at the same fold count, so a recipient's fold is its # twin's fold; the twins the training rows hold are checked, so a cell # value that moved between twin and copy cannot put a twin's forest on - # its copy. + # its copy. A twin is found by its twin identity, so the check reaches + # the copies of every geographic clone, folded or not. recipient_folds, _ = cell_fold_ids( recipient_predictors, folds=folds, seed=seed, used=used ) twin_folds = pd.Series(training_folds, index=training_identities) if not twin_folds.index.is_unique: raise ValueError("training identities must be unique (one row per twin).") - twins = twin_folds.reindex(identities) + twins = twin_folds.reindex(twin_identities) known = twins.notna().to_numpy() if known.any() and not np.array_equal( twins.to_numpy()[known].astype(np.int64), recipient_folds[known] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 0a1840abff..ca7b29144c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -45,6 +45,7 @@ from microcosm.build.uk_runtime.hmrc_source_contract import ( HMRC_DISTRIBUTIONAL_INPUTS, ) +from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_FOLD_EVIDENCE from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, @@ -515,6 +516,10 @@ def evidence(self) -> dict[str, object]: evidence["donor_frame"] = dict(self.imputation.donor_frame) if self.imputation.gate_levels: evidence[UK_GATE_LEVELS_EVIDENCE] = dict(self.imputation.gate_levels) + # How the two forests' fold met their survey records, on a frame of + # more than one geographic clone only (microcosm#931). + if self.imputation.geographic_fold is not None: + evidence[GEOGRAPHIC_FOLD_EVIDENCE] = dict(self.imputation.geographic_fold) return evidence @@ -746,6 +751,8 @@ def checkpoint_metadata(self) -> dict[str, object]: return { "evidence": { "stage": "spi_support_channel", + # The frame's geographic clone count (microcosm#931). + "clone_count": result.clone_count, # The copy count is exact: the stage gate reads it, and it # counts a draw over every household, not an attribute cell. "spi_households": result.n_spi_households, @@ -769,10 +776,8 @@ def checkpoint_metadata(self) -> dict[str, object]: result.pension_age_spi_prior_mass_share ), "state_pension_age": result.state_pension_age, - "pension_age_allocation": ( - None - if result.pension_age_allocation is None - else dict(result.pension_age_allocation) + "pension_age_allocation": _pension_age_allocation_receipt( + result.pension_age_allocation ), "household_weight_kind": ( result.household_weight_kind.value @@ -783,6 +788,36 @@ def checkpoint_metadata(self) -> dict[str, object]: } +#: The pension-age allocation's counts of survey households, which the stage's +#: receipt rounds: the pension-age base households' distinct sources and clone +#: 0's pension-age SPI copies' (microcosm#931). +_PENSION_AGE_ALLOCATION_COUNTS = ( + "pension_age_households", + "pension_age_spi_households", +) + + +def _pension_age_allocation_receipt( + allocation: Mapping[str, object] | None, +) -> dict[str, object] | None: + """The pension-age allocation as the receipt gives it. + + Its two counts of distinct source households rounded by the receipts' + rule (``receipt_counts.rounded_count``), its masses as they are. + """ + + if allocation is None: + return None + return { + key: ( + rounded_count(int(value)) + if key in _PENSION_AGE_ALLOCATION_COUNTS + else value + ) + for key, value in allocation.items() + } + + @dataclass(frozen=True) class UKSPIIncomeSpineStageTransform: spi_tab_path: Path diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py index 1a92a97ef7..df98ffa313 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py @@ -288,6 +288,10 @@ class UKSPISupportResult: expected_spi_households: float | None = None spi_source_households: int | None = None spi_inclusion_probability: float | None = None + #: The frame's geographic clone count ``K``, which the prior allocation + #: reads from ``household_clone_index``: one more than the largest index, + #: one on a household table without the column (microcosm#931). + clone_count: int = 1 @property def n_spi_households(self) -> int: @@ -563,6 +567,7 @@ def replace_uk_spi_support_tables( mass_log=allocated.mass_log, replaced_spi_households=int(synthetic.sum()), spi_prior_mass_share=share, + clone_count=allocated.clone_count, ) @@ -1236,9 +1241,27 @@ def _allocate_spi_prior_mass( pension_age_allocation = None if pension_age_share is not None: final = allocated_weights.values + # The two counts are of survey households: the base households' true + # sources, which every geographic clone shares, and those of clone 0's + # SPI copies, which a one-clone spine copies alike (the share draws + # each clone on its own ids, so the copies of every clone together + # grow with the clone count). Both are the same at every clone count; + # the stage's receipt rounds them (microcosm#931). Each counts the + # carriers of the mass printed beside it, as every receipt counts + # them: a source on a row of positive weight. + sources = household["source_household_id"].to_numpy() + clone_zero = ( + np.ones(len(household), dtype=bool) + if clone_strata is None + else clone_strata == 0 + ) pension_age_allocation = { - "pension_age_households": int((pension_age & base_mask).sum()), - "pension_age_spi_households": int((pension_age & spi_mask).sum()), + "pension_age_households": _distinct_sources( + sources, pension_age & base_mask, pre_weights + ), + "pension_age_spi_households": _distinct_sources( + sources, pension_age & spi_mask & clone_zero, final + ), "pension_age_prior_mass": float(pre_weights[pension_age].sum()), "pension_age_spi_mass": float(final[pension_age & spi_mask].sum()), "other_prior_mass": float(pre_weights[~pension_age].sum()), @@ -1258,9 +1281,28 @@ def _allocate_spi_prior_mass( expected_spi_households=result.expected_spi_households, spi_source_households=result.spi_source_households, spi_inclusion_probability=result.spi_inclusion_probability, + clone_count=( + 1 + if clone_strata is None or not clone_strata.size + else int(clone_strata.max()) + 1 + ), ) +def _distinct_sources( + sources: np.ndarray, mask: np.ndarray, weights: np.ndarray +) -> int: + """The distinct source households carrying the rows ``mask`` selects. + + A carrier is the receipts' (``receipt_counts.distinct_carriers``): a + source household that is not missing, on a row of positive weight, so a + row of zero weight adds nothing to the count. + """ + + selected = np.asarray(mask, dtype=bool) & (np.asarray(weights, dtype=float) > 0.0) + return int(pd.Series(sources[selected]).dropna().nunique()) + + def _geographic_clone_strata(household: pd.DataFrame) -> np.ndarray | None: """Each household's geographic clone, or None without the index column. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index f89636b609..10f8d8a495 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -9,6 +9,7 @@ from microcosm.build.gates import GateResult from microcosm.build.stochastic_assignment import binomial_share_tolerance +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.receipt_counts import ( UK_BELOW_TEN_BAND, UK_DISCLOSURE_FLOOR, @@ -25,8 +26,13 @@ def uk_stage_health_gate( stage: str, check: str, parameters: Mapping[str, object], + frame: object | None = None, ) -> GateResult: - """Evaluate one spine-stage receipt against spec-declared thresholds.""" + """Evaluate one spine-stage receipt against spec-declared thresholds. + + ``frame`` is the gate phase's frame, which only the band donor check + reads: it recomputes a capped seating's seats there (microcosm#931). + """ if evidence.get("stage") not in {stage, _legacy_receipt_stage(stage)}: return GateResult( @@ -72,7 +78,9 @@ def uk_stage_health_gate( if check == "high_value_home_support": return _high_value_home_support_gate(stage, evidence, parameters) if check == "spi_income_band_donor_support": - return _spi_income_band_donor_support_gate(stage, evidence, parameters) + return _spi_income_band_donor_support_gate( + stage, evidence, parameters, frame=frame + ) if check == "cgt_imputation_summary": return _cgt_imputation_summary_gate(stage, evidence, parameters) if check == "cgt_asset_type_summary": @@ -177,14 +185,54 @@ def _mapping(value: object, *, label: str) -> Mapping[str, object]: return value +def _receipt_clone_count(receipt: Mapping[str, object], *, label: str) -> int: + """The geographic clone count ``K`` a receipt gives, or 1 where it gives none. + + A stage whose checked row counts grow with the clone count records + ``clone_count`` only when ``K`` is more than one (microcosm#931), so a + one-clone receipt keeps its form and its check reads the declared + allowances as they stand. + """ + + clone_count = receipt.get("clone_count", 1) + if ( + not isinstance(clone_count, int) + or isinstance(clone_count, bool) + or clone_count < 1 + ): + raise ValueError(f"{label}.clone_count must be a positive integer.") + return clone_count + + +def _rows_allowed(allowance: object, clone_count: int) -> str: + """A row allowance as a failure prints it: as declared at one clone, and + at ``K`` clones as the total, then the declared figure for each clone.""" + + if clone_count == 1: + return f"{allowance}" + return ( + f"{int(allowance) * clone_count}, {allowance} for each of the " + f"{clone_count} geographic clones" + ) + + def _support_clip_gate( stage: str, evidence: Mapping[str, object], parameters: Mapping[str, object], ) -> GateResult: + """Every declared column's clip within its row allowances. + + The allowances are declared for one geographic clone. On ``K`` clones + the stage clips every clone's rows, so each allowance is multiplied by + the receipt's ``clone_count`` (microcosm#931); a receipt without one is + one clone. + """ + check = "support_clip" clip = _mapping(evidence.get("support_clip"), label=f"{stage}.support_clip") columns = _mapping(clip.get("columns"), label=f"{stage}.support_clip.columns") + clone_count = _receipt_clone_count(clip, label=f"{stage}.support_clip") expected_columns = tuple(str(column) for column in parameters["columns"]) exempt_columns = {str(column) for column in parameters.get("exempt_columns", ())} max_low = _mapping( @@ -221,28 +269,32 @@ def _support_clip_gate( f"{stage}: {column!r} declares no clipped_low_rows allowance; " "pin one at the receipted baseline or exempt the column." ) - elif low_rows > int(allowed_low): + elif low_rows > int(allowed_low) * clone_count: failures.append( - f"{stage}: {column!r} clipped_low_rows {low_rows} exceeds {allowed_low}." + f"{stage}: {column!r} clipped_low_rows {low_rows} exceeds " + f"{_rows_allowed(allowed_low, clone_count)}." ) if allowed_high is None: failures.append( f"{stage}: {column!r} declares no clipped_high_rows allowance; " "pin one at the receipted baseline or exempt the column." ) - elif high_rows > int(allowed_high): + elif high_rows > int(allowed_high) * clone_count: failures.append( - f"{stage}: {column!r} clipped_high_rows {high_rows} exceeds {allowed_high}." + f"{stage}: {column!r} clipped_high_rows {high_rows} exceeds " + f"{_rows_allowed(allowed_high, clone_count)}." ) if "donor_min" in receipt and "donor_max" in receipt: lower = _finite_number(receipt["donor_min"], label=f"{column}.donor_min") upper = _finite_number(receipt["donor_max"], label=f"{column}.donor_max") if lower > upper: failures.append(f"{stage}: {column!r} donor_min exceeds donor_max.") - details = { + details: dict[str, object] = { "columns_checked": len(expected_columns) - len(exempt_columns), "exempt_columns": sorted(exempt_columns), } + if clone_count > 1: + details["clone_count"] = clone_count # The donor floor (microcosm#1063 c9): negative diary consumption is raised # to the declared floor before the clip ranges are read, so no clip range # may start below it and no negative donor row may remain. @@ -2024,12 +2076,44 @@ def _cgt_support_split_gate( #: The two forms the band donor receipt records its seating scale in: #: ``spi_band_donors.SPI_INCOME_BAND_FULL_SEATING`` and ``..._CAPPED_SEATING``. _BAND_DONOR_SEATINGS = ("full", "capped") +#: The band donor stage's columns, spelled here so the gate imports no stage +#: module (``spi_band_donors``' names; a test holds the spellings equal): its +#: donor flag and band on the household, its carrier flag on the person, and +#: the geographic clone index every household of a clone-first spine carries. +_BAND_DONOR_FLAG_COLUMN = "household_is_spi_income_band_donor" +_BAND_DONOR_BAND_COLUMN = "spi_income_band_donor_lower_bound" +_BAND_CARRIER_FLAG_COLUMN = "person_is_spi_income_band_carrier" +_BAND_DONOR_CLONE_INDEX_COLUMN = "household_clone_index" +#: The copy-id field the band donors take (``spi_support. +#: SPI_SUPPORT_COPY_ID_FIELD``), and the fields minted after it. Every later +#: layer keeps each row it copies at its own id and mints the copies at +#: ordinals from one, so a band donor's seat is its row whose id holds no +#: ordinal in any later field (microcosm#931). +_BAND_DONOR_COPY_ID_FIELD = "spi_support_channel" +_COPY_ID_FIELDS_AFTER_BAND_DONORS = tuple( + field.name + for field in copy_id_table.UK_COPY_ID_FIELDS + if field.radix > copy_id_table.radix(_BAND_DONOR_COPY_ID_FIELD) +) +#: The funding the receipt states in two booleans, each with what it says. +_BAND_DONOR_FUNDING_BOOLEANS = ( + ( + "funding_within_share", + "every funding stratum keeps at least one less the declared reallocation share", + ), + ( + "funding_matches_bands", + "the bands and the funding strata each carry the reallocated mass", + ), +) def _spi_income_band_donor_support_gate( stage: str, evidence: Mapping[str, object], parameters: Mapping[str, object], + *, + frame: object | None = None, ) -> GateResult: """Every reserved band carries its donors, and their mass was reallocated. @@ -2046,6 +2130,21 @@ def _spi_income_band_donor_support_gate( band's donors split evenly across the clones, and at full seating the plan and the minimum are ``K`` times one clone's. The band's published mass is checked as on one clone. + + The receipt states the funding in two booleans at every seating + (microcosm#931): ``funding_within_share``, every funding stratum keeping + at least one less the stage's declared reallocation share, and + ``funding_matches_bands``, the bands and the funding strata each carrying + the reallocated mass. Both must be true, and one less that share must + reach the gate's minimum funding factor. A full seating seats each band's + plan, and the gate reads its counts from the receipt. A capped seating's + receipt gives no exact count, so the gate recomputes its seats from + ``frame``, the transferred phase's frame + (:func:`_band_donor_seat_failures`), and its details carry only ``K``, + the seating, the mass scale, the booleans and the count of bands. On a + capped seating the bands' mass is the stage's ``funding_matches_bands``: + later copy layers reweight the seats, so the frame cannot restate it. No + failure text gives a count. """ check = "spi_income_band_donor_support" @@ -2074,7 +2173,7 @@ def _spi_income_band_donor_support_gate( value = _finite_number(evidence.get(key), label=f"{stage}.{key}") if value != expected: failures.append( - f"{stage}: receipt {key} {value} differs from the gate's {expected}." + f"{stage}: receipt {key} differs from the gate's declared value." ) clone_count = evidence.get("clone_count") if ( @@ -2083,17 +2182,8 @@ def _spi_income_band_donor_support_gate( or clone_count < 1 ): raise ValueError(f"{stage}.clone_count must be a positive integer.") - over_clones = ( - "" - if clone_count == 1 - else f" divided by the geographic clone count {clone_count}" - ) - minimum_seated = minimum_donors * clone_count - per_clone_minimum = ( - "" - if clone_count == 1 - else f" ({minimum_donors} in each of the {clone_count} geographic clones)" - ) + over_clones = "" if clone_count == 1 else " divided by the geographic clone count" + per_clone_minimum = "" if clone_count == 1 else " in each geographic clone" # The receipt records the seating scale as full or capped: an exact scale # below one gives back the eligible household count, so the gate names # any other value by its type alone. @@ -2105,7 +2195,9 @@ def _spi_income_band_donor_support_gate( ) mass_scale = _finite_number(evidence.get("mass_scale"), label=f"{stage}.mass_scale") if not 0.0 < mass_scale <= 1.0: - failures.append(f"{stage}: mass_scale {mass_scale} is outside (0, 1].") + failures.append( + f"{stage}: mass_scale is not a share above zero and at most one." + ) full_seating = seating == "full" full_scale = full_seating and mass_scale == 1.0 sample_fraction = _finite_number( @@ -2114,7 +2206,8 @@ def _spi_income_band_donor_support_gate( if sample_fraction == 1.0 and not full_scale: failures.append( f"{stage}: a full-sample build must seat and fund the donors at full " - f"scale, got seating_scale {seating} and mass_scale {mass_scale}." + f"scale, got seating_scale {seating} and a mass_scale " + f"{'of one' if mass_scale == 1.0 else 'other than one'}." ) mass = _mapping(evidence.get("mass"), label=f"{stage}.mass") @@ -2126,9 +2219,8 @@ def _spi_income_band_donor_support_gate( effective_tolerance = max(tolerance, _FLOAT_RELATIVE_TOLERANCE) if deviation > effective_tolerance: failures.append( - f"{stage}: household mass deviation {deviation} exceeds the effective " - f"tolerance {effective_tolerance}; the donors must be funded from the " - "incumbent households, not added." + f"{stage}: household mass moved by more than the declared deviation; " + "the donors must be funded from the incumbent households, not added." ) bands = evidence.get("bands") @@ -2146,115 +2238,140 @@ def _spi_income_band_donor_support_gate( _finite_number(row.get("lower_bound"), label=f"{stage}.lower_bound") ) seen.append(lower) - donors = int( - _finite_number( - row.get("donor_households"), label=f"{stage}.donor_households" - ) - ) - expected_donors = int( - _finite_number( - row.get("expected_donor_households"), - label=f"{stage}.expected_donor_households", - ) - ) - carriers = int(_finite_number(row.get("carriers"), label=f"{stage}.carriers")) weight = _finite_number(row.get("donor_weight"), label=f"{stage}.donor_weight") - published = _finite_number( - row.get("published_taxpayers"), label=f"{stage}.published_taxpayers" - ) - donor_total += donors - donor_mass += weight * donors heaviest_donor = max(heaviest_donor, weight) - if donors <= 0: - failures.append(f"{stage}: band from {lower} seats no donor.") - if donors != carriers: - failures.append( - f"{stage}: band from {lower} has {donors} donors but {carriers} carriers." + # A full seating's counts are the plan's, read from the receipt; a + # capped seating's are recomputed from the frame below. + donors = carriers = expected_donors = 0 + published = 0.0 + if full_seating: + donors = int( + _finite_number( + row.get("donor_households"), label=f"{stage}.donor_households" + ) ) - if weight <= 0.0: - failures.append( - f"{stage}: band from {lower} donor weight {weight} is not positive." + expected_donors = int( + _finite_number( + row.get("expected_donor_households"), + label=f"{stage}.expected_donor_households", + ) ) + carriers = int( + _finite_number(row.get("carriers"), label=f"{stage}.carriers") + ) + published = _finite_number( + row.get("published_taxpayers"), label=f"{stage}.published_taxpayers" + ) + donor_total += donors + donor_mass += weight * donors + if donors <= 0: + failures.append(f"{stage}: band from {lower} seats no donor.") + if donors != carriers: + failures.append( + f"{stage}: band from {lower}'s seats do not each hold one carrier." + ) + if weight <= 0.0: + failures.append(f"{stage}: band from {lower} donor weight is not positive.") if clone_count * weight > maximum_donor_weight * ( 1.0 + _FLOAT_RELATIVE_TOLERANCE ): failures.append( - f"{stage}: band from {lower} donor weight {weight} exceeds the " - f"maximum {maximum_donor_weight}{over_clones}." + f"{stage}: band from {lower} donor weight exceeds the maximum" + f"{over_clones}." ) + if not full_seating: + continue if donors % clone_count: failures.append( - f"{stage}: band from {lower} seats {donors} donors, which do not " - f"split evenly across the {clone_count} geographic clones." + f"{stage}: band from {lower}'s donors do not split evenly across " + "the geographic clones." ) - if full_seating and (donors != expected_donors or donors < minimum_seated): + if donors != expected_donors or donors < minimum_donors * clone_count: failures.append( - f"{stage}: band from {lower} seats {donors} donors at full scale; " - f"the plan seats {expected_donors} and the minimum is " - f"{minimum_seated}{per_clone_minimum}." + f"{stage}: band from {lower} seats other than its plan at full " + f"scale, or fewer than the minimum{per_clone_minimum}." ) # A scaled frame seats fewer donors or lighter ones, so its bands fall # short of the published mass by construction; the receipt shows it. if full_scale and abs(weight * donors - published) > taxpayer_tolerance: failures.append( - f"{stage}: band from {lower} weighted taxpayers " - f"{weight * donors} differ from the published {published}." + f"{stage}: band from {lower} weighted taxpayers differ from the " + "published count." ) if sorted(seen) != sorted(expected_bands): failures.append( f"{stage}: bands {sorted(seen)} differ from the declared {sorted(expected_bands)}." ) - donor_count = int( - _finite_number(evidence.get("donor_count"), label=f"{stage}.donor_count") - ) - if donor_count != donor_total: - failures.append( - f"{stage}: receipt donor_count {donor_count} differs from the bands' " - f"{donor_total}." + full_details: dict[str, object] = {} + if full_seating: + donor_count = int( + _finite_number(evidence.get("donor_count"), label=f"{stage}.donor_count") ) - - funding = evidence.get("funding") - if not isinstance(funding, list | tuple) or not funding: - raise ValueError(f"{stage}.funding must be a non-empty list.") - funded_mass = 0.0 - smallest_factor = 1.0 - for row in funding: - if not isinstance(row, Mapping): - failures.append(f"{stage}: funding row is not an object.") - continue - factor = _finite_number(row.get("factor"), label=f"{stage}.funding.factor") - funded_mass += _finite_number( - row.get("donor_mass"), label=f"{stage}.funding.donor_mass" + if donor_count != donor_total: + failures.append(f"{stage}: receipt donor_count differs from the bands'.") + reallocated = _finite_number( + evidence.get("reallocated_mass"), label=f"{stage}.reallocated_mass" ) - smallest_factor = min(smallest_factor, factor) - if not ( - minimum_funding_factor * (1.0 - _FLOAT_RELATIVE_TOLERANCE) <= factor <= 1.0 + if abs(donor_mass - reallocated) > effective_tolerance * max( + abs(reallocated), 1.0 ): failures.append( - f"{stage}: funding stratum {row.get('stratum')!r} factor {factor} " - f"is outside [{minimum_funding_factor}, 1]." + f"{stage}: the bands carry other donor mass than the receipt " + "reallocated." + ) + full_details = { + "donor_count": donor_count, + "reallocated_mass": reallocated, + "relative_mass_deviation": deviation, + "heaviest_donor_weight": heaviest_donor, + } + else: + failures.extend( + _band_donor_seat_failures( + stage, frame, bands=expected_bands, clone_count=clone_count ) - reallocated = _finite_number( - evidence.get("reallocated_mass"), label=f"{stage}.reallocated_mass" + ) + + # The funding rows carry each stratum and its rounded donors; the receipt + # states the funding in its two booleans, and the gate holds the stage's + # declared share to its minimum funding factor (microcosm#931). + funding = evidence.get("funding") + if not isinstance(funding, list | tuple) or not funding: + raise ValueError(f"{stage}.funding must be a non-empty list.") + if any(not isinstance(row, Mapping) for row in funding): + failures.append(f"{stage}: funding row is not an object.") + booleans: dict[str, object] = {} + for key, meaning in _BAND_DONOR_FUNDING_BOOLEANS: + value = evidence.get(key) + booleans[key] = value if isinstance(value, bool) else None + if value is None: + failures.append(f"{stage}: the receipt carries no {key}: {meaning}.") + elif value is not True: + failures.append(f"{stage}: the receipt's {key} is not true: {meaning}.") + share = _finite_number( + evidence.get("maximum_reallocation_share"), + label=f"{stage}.maximum_reallocation_share", ) - for label, value in (("bands", donor_mass), ("funding strata", funded_mass)): - if abs(value - reallocated) > effective_tolerance * max(abs(reallocated), 1.0): - failures.append( - f"{stage}: the {label} carry donor mass {value}, the receipt " - f"reallocated {reallocated}." - ) + if not 0.0 < share < 1.0: + failures.append( + f"{stage}: the receipt's maximum_reallocation_share is not a share " + "strictly between zero and one." + ) + elif 1.0 - share < minimum_funding_factor * (1.0 - _FLOAT_RELATIVE_TOLERANCE): + failures.append( + f"{stage}: the receipt's maximum_reallocation_share lets a funding " + "stratum keep less than the gate's minimum funding factor." + ) details = { "bands_checked": len(bands), - "donor_count": donor_count, "clone_count": clone_count, "seating_scale": seating, "mass_scale": mass_scale, - "reallocated_mass": reallocated, - "relative_mass_deviation": deviation, - "heaviest_donor_weight": heaviest_donor, - "smallest_funding_factor": smallest_factor, - "funding_strata": len(funding), + **booleans, + **full_details, } + if full_seating: + details["funding_strata"] = len(funding) return ( _fail(stage, check, failures, details) if failures @@ -2262,6 +2379,100 @@ def _spi_income_band_donor_support_gate( ) +def _band_donor_seat_failures( + stage: str, + frame: object | None, + *, + bands: list[int], + clone_count: int, +) -> list[str]: + """What a capped seating's seats, recomputed from ``frame``, fail. + + The seats are the band donor rows that are originals in every copy-id + field minted after the band donors (:data:`_COPY_ID_FIELDS_AFTER_BAND_DONORS`), + so a later layer's copy of a donor is not read as a seat. Every declared + band must seat a donor, every seat must sit in a declared band and hold + exactly one carrier (``person_is_spi_income_band_carrier`` on the seat's + own people), and each band's seats must split evenly across the + receipt's ``clone_count`` geographic clones, read from + ``household_clone_index`` (a frame without it is one clone). Each failure + names a band or a column, never a count (microcosm#931). + """ + + if frame is None: + return [ + f"{stage}: a capped seating's seats are recomputed from the phase's " + "frame, and the gate was given none." + ] + household = frame.table("household") + person = frame.table("person") + missing = [ + f"{entity}.{column}" + for entity, table, column in ( + ("household", household, "household_id"), + ("household", household, _BAND_DONOR_FLAG_COLUMN), + ("household", household, _BAND_DONOR_BAND_COLUMN), + ("person", person, "person_household_id"), + ("person", person, _BAND_CARRIER_FLAG_COLUMN), + ) + if column not in table.columns + ] + if missing: + return [ + f"{stage}: the frame carries no {', '.join(missing)}, so the capped " + "seating's seats cannot be recomputed." + ] + ids = np.asarray(household["household_id"]) + clones = ( + np.asarray(household[_BAND_DONOR_CLONE_INDEX_COLUMN]) + if _BAND_DONOR_CLONE_INDEX_COLUMN in household.columns + else np.zeros(len(household), dtype=np.int64) + ) + for label, values in (("household ids", ids), ("geographic clone indices", clones)): + if values.dtype.kind not in "iu" or bool((values < 0).any()): + return [ + f"{stage}: the frame's {label} are not non-negative integers, so " + "the capped seating's seats cannot be recomputed." + ] + clones = clones.astype(np.int64, copy=False) + seat = np.array(household[_BAND_DONOR_FLAG_COLUMN], dtype=bool) + for field in _COPY_ID_FIELDS_AFTER_BAND_DONORS: + seat &= copy_id_table.ordinal(ids, field) == 0 + seat_ids = ids[seat] + seat_bands = np.array(household[_BAND_DONOR_BAND_COLUMN], dtype=float)[seat] + seat_clones = clones[seat] + # Each seat's carriers: the flagged people whose household is the seat. + carried = np.array(person[_BAND_CARRIER_FLAG_COLUMN], dtype=bool) + carrier_households = np.asarray(person["person_household_id"])[carried] + order = np.argsort(seat_ids, kind="stable") + sorted_ids = seat_ids[order] + position = np.searchsorted(sorted_ids, carrier_households) + on_seat = position < sorted_ids.size + on_seat[on_seat] = sorted_ids[position[on_seat]] == carrier_households[on_seat] + carriers_per_seat = np.empty(sorted_ids.size, dtype=np.int64) + carriers_per_seat[order] = np.bincount(position[on_seat], minlength=sorted_ids.size) + + failures: list[str] = [] + if not bool(np.isin(seat_bands, np.asarray(bands, dtype=float)).all()): + failures.append(f"{stage}: a seat holds a band the gate does not declare.") + for band in bands: + in_band = seat_bands == float(band) + if not in_band.any(): + failures.append(f"{stage}: band from {band} seats no donor.") + continue + if bool((carriers_per_seat[in_band] != 1).any()): + failures.append( + f"{stage}: band from {band}'s seats do not each hold one carrier." + ) + per_clone = np.bincount(seat_clones[in_band], minlength=clone_count) + if per_clone.size != clone_count or bool((per_clone != per_clone[0]).any()): + failures.append( + f"{stage}: band from {band}'s donors do not split evenly across " + "the geographic clones." + ) + return failures + + def _cgt_imputation_summary_gate( stage: str, evidence: Mapping[str, object], @@ -4513,10 +4724,13 @@ def _property_components_gate( of its two boundaries (one straddler for an empty band), capped at twice the largest landlord weight. The open top band's profit multiplier stays below its declared cap; a multiplier at its floor of 1 is reported, not failed. + The two row maxima are declared for one geographic clone; on ``K`` clones + they are multiplied by the receipt's ``clone_count`` (microcosm#931). """ check = "property_components" receipts = _mapping(evidence.get("receipts"), label=f"{stage}.receipts") + clone_count = _receipt_clone_count(evidence, label=stage) failures: list[str] = [] maximum_below = int(parameters["maximum_rows_receipts_below_profit"]) below = int( @@ -4525,10 +4739,10 @@ def _property_components_gate( label=f"{stage}.receipts.rows_receipts_below_profit", ) ) - if below > maximum_below: + if below > maximum_below * clone_count: failures.append( f"{stage}: {below} rows carry receipts below profit " - f"(allowed {maximum_below})." + f"(allowed {_rows_allowed(maximum_below, clone_count)})." ) maximum_above = int(parameters["maximum_rows_with_profit_above_band"]) above = int( @@ -4537,10 +4751,10 @@ def _property_components_gate( label=f"{stage}.receipts.rows_with_profit_above_their_band", ) ) - if above > maximum_above: + if above > maximum_above * clone_count: failures.append( f"{stage}: {above} landlords sit in a receipts band below their " - f"profit (allowed {maximum_above})." + f"profit (allowed {_rows_allowed(maximum_above, clone_count)})." ) factor = _finite_number( parameters["maximum_band_error_in_boundary_weights"], @@ -4598,7 +4812,7 @@ def _property_components_gate( f"{stage}: the top band's profit multiplier {multiplier} reached its " f"cap {cap}; the receipts total is out of reach." ) - details = { + details: dict[str, object] = { "rows_receipts_below_profit": below, "rows_with_profit_above_their_band": above, "largest_landlord_weight": largest, @@ -4607,6 +4821,8 @@ def _property_components_gate( "top_band_multiplier_cap": cap, "top_band_multiplier_at_floor": bool(top.get("multiplier_at_floor", False)), } + if clone_count > 1: + details["clone_count"] = clone_count if failures: return _fail(stage, check, failures, details) return _pass(stage, check, details) @@ -4819,6 +5035,8 @@ def _regional_home_values_gate( weighted value stays positive on both sides of the stage. A count the receipt bands for resting on fewer than three records is flagged rather than failed where the band cannot tell (María's ruling of 2026-10-26). + The stage counts the sources at the cap in geographic clone 0's rows, so + the declared count holds at any clone count (microcosm#931). """ check = "regional_home_values" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/support_clip.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/support_clip.py index 1935b84da9..392056026c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/support_clip.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/support_clip.py @@ -1,4 +1,12 @@ -"""Shared support-clip receipts for UK donor imputation stages.""" +"""Shared support-clip receipts for UK donor imputation stages. + +A receipt gives each clipped column's donor range and the rows the clip +raised or lowered into it. On a frame of ``K`` geographic clones +(microcosm#931) every clone's rows are clipped, so the receipt also gives +``clone_count``, only when ``K`` is more than one, and the support-clip +check allows ``K`` times each declared row allowance; a one-clone receipt +keeps its form. +""" from __future__ import annotations @@ -13,13 +21,19 @@ class UKSupportClipReceipt: stage: str columns: Mapping[str, Mapping[str, object]] + #: The frame's geographic clone count ``K``; the evidence gives it only + #: when it is more than one. + clone_count: int = 1 def evidence(self) -> dict[str, object]: - return { + evidence: dict[str, object] = { "columns": { column: dict(receipt) for column, receipt in self.columns.items() } } + if self.clone_count > 1: + evidence["clone_count"] = self.clone_count + return evidence @dataclass(frozen=True) @@ -35,9 +49,23 @@ def support_clip_to_donor_with_receipt( columns: Sequence[str], stage: str, exempt: set[str] | None = None, + clone_count: int = 1, ) -> UKSupportClipResult: - """Clip draws to donor support and return a structured receipt.""" + """Clip draws to donor support and return a structured receipt. + + ``clone_count`` is the frame's geographic clone count + (``geographic_clones.geographic_clone_count``), which the receipt carries + for the support-clip check's row allowances. + """ + if ( + isinstance(clone_count, bool) + or not isinstance(clone_count, int | np.integer) + or clone_count < 1 + ): + raise ValueError( + f"{stage}: the support clip's clone_count must be a positive integer." + ) clipped = draws.copy() exempt = exempt or set() receipts: dict[str, dict[str, object]] = {} @@ -68,5 +96,7 @@ def support_clip_to_donor_with_receipt( clipped[column] = clipped[column].clip(lower=lower, upper=upper) return UKSupportClipResult( clipped=clipped, - receipt=UKSupportClipReceipt(stage=stage, columns=receipts), + receipt=UKSupportClipReceipt( + stage=stage, columns=receipts, clone_count=int(clone_count) + ), ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_capital_coherence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_capital_coherence.py index cb10e54aba..22cc9612f0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_capital_coherence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_capital_coherence.py @@ -24,7 +24,11 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.uk_runtime.frs_spine import UC_CAPITAL_UNAVAILABLE -from microcosm.build.uk_runtime.household_lineage import fold_geographic_clones +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_FOLD_EVIDENCE, + fold_geographic_clones, + geographic_fold_receipt, +) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, @@ -44,6 +48,7 @@ ) from microcosm.frame import Frame +UC_CAPITAL_COHERENCE_STAGE_NAME = "uc_capital_coherence" UC_CAPITAL_REDRAW_OUTPUT = "frs_benunit_capital" UC_CAPITAL_REDRAW_SEED = 0 UC_CAPITAL_REDRAW_SALT = UC_CAPITAL_REDRAW_OUTPUT @@ -223,12 +228,15 @@ class UKUCCapitalCoherenceResult: default_factory=dict ) property_shares: Mapping[str, object] = field(default_factory=dict) + #: How the capital redraw's donor fold met its survey records, on a frame + #: of more than one geographic clone only (microcosm#931). + geographic_fold: Mapping[str, object] | None = None def evidence(self) -> dict[str, object]: """Return JSON-safe stage evidence.""" - return { - "stage": "uc_capital_coherence", + evidence: dict[str, object] = { + "stage": UC_CAPITAL_COHERENCE_STAGE_NAME, "post_fill_reporter_count": self.post_fill_reporter_count, "redrawn_spi_reporter_count": self.redrawn_spi_reporter_count, "redrawn_spi_benefit_units": self.redrawn_spi_benefit_units, @@ -242,6 +250,9 @@ def evidence(self) -> dict[str, object]: ), "property_shares": dict(self.property_shares), } + if self.geographic_fold is not None: + evidence[GEOGRAPHIC_FOLD_EVIDENCE] = dict(self.geographic_fold) + return evidence @dataclass(frozen=True) @@ -349,8 +360,9 @@ def cohere_uc_capital( ) redraw = spi levels: dict[str, int] = dict.fromkeys(_UC_CAPITAL_LEVELS, 0) + geographic_fold: dict[str, object] | None = None if redraw.any(): - levels = _redraw_spi_capital( + levels, geographic_fold = _redraw_spi_capital( benunit, person=person, household=household, @@ -443,6 +455,7 @@ def cohere_uc_capital( "after": receipt_after, }, property_shares=property_receipt, + geographic_fold=geographic_fold, ) @@ -645,7 +658,7 @@ def _redraw_spi_capital( capital: np.ndarray, investment: np.ndarray, minimum_cell_donors: int = UC_CAPITAL_MINIMUM_CELL_DONORS, -) -> dict[str, int]: +) -> tuple[dict[str, int], dict[str, object] | None]: """Redraw ``redraw`` units' capital from base donors in their cell. A cell is reporter status x investment-income band x couple x @@ -659,7 +672,9 @@ def _redraw_spi_capital( microcosm#931). A cell's donors are counted in survey records, the distinct replica keys among its rows, so neither identical nor differing clones grow the count, while every other copy counts as a record of its - own. Returns the number of redrawn units drawn at each level. + own. Returns the number of redrawn units drawn at each level, and the + fold's receipt on a frame of more than one geographic clone + (``household_lineage.geographic_fold_receipt``; None on one clone). """ if not isinstance(minimum_cell_donors, int) or minimum_cell_donors <= 0: @@ -687,6 +702,12 @@ def _redraw_spi_capital( } ), ) + geographic_fold = geographic_fold_receipt( + {"person": person, "benunit": benunit, "household": household}, + "benunit", + fold, + stage=UC_CAPITAL_COHERENCE_STAGE_NAME, + ) donor, donor_weight = fold.kept, fold.weights target_ids = benunit["benunit_id"].to_numpy() draws = stable_identity_uniforms( @@ -764,7 +785,7 @@ def _redraw_spi_capital( selected = np.searchsorted(cdf, draws[target_cell], side="right") capital[target_cell] = donor_values[np.minimum(selected, len(cdf) - 1)] levels[level] += int(target_cell.sum()) - return levels + return levels, geographic_fold def _survey_records(key_group: np.ndarray, pool: np.ndarray) -> int: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_reporter_redraw.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_reporter_redraw.py index 3bd5cf1abb..20be199f4f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_reporter_redraw.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/uc_reporter_redraw.py @@ -21,7 +21,11 @@ integer_record_keys, stable_identity_uniforms, ) -from microcosm.build.uk_runtime.household_lineage import collapse_geographic_clones +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_FOLD_EVIDENCE, + fold_geographic_clones, + geographic_fold_receipt, +) from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, @@ -149,6 +153,9 @@ class UKUCReporterRedrawResult: reporter_transitions: Mapping[str, Mapping[str, Mapping[str, int]]] #: The redraw's gate outcome, when the gate is not in-sample. gate_outcomes: Mapping[str, str] | None = None + #: How the forest's geographic fold met its survey records + #: (``household_lineage.geographic_fold_receipt``); None on one clone. + geographic_fold: Mapping[str, object] | None = None def evidence(self) -> dict[str, object]: """Return JSON-safe stage evidence.""" @@ -169,6 +176,8 @@ def evidence(self) -> dict[str, object]: } if self.gate_outcomes: evidence[UK_GATE_OUTCOMES_EVIDENCE] = dict(self.gate_outcomes) + if self.geographic_fold is not None: + evidence[GEOGRAPHIC_FOLD_EVIDENCE] = dict(self.geographic_fold) return evidence @@ -321,14 +330,15 @@ def redraw_spi_reported_uc( ) # The forest trains on each screened benefit unit once: its geographic # clones that agree on the predictors and the reported amount fold into - # its first row with their weight, clones that differ (a member's + # its lowest-id clone with their weight, clones that differ (a member's # property income drawn on its own id, say) stay rows of their own, and # every other copy stays its own row; a frame without geographic clones - # keeps every row and weight (household_lineage.collapse_geographic_clones, + # keeps every row and weight (household_lineage.fold_geographic_clones, # microcosm#931). The screen is each row's own engine output, so the fold - # follows it. - training, fit_weights = collapse_geographic_clones( - {"person": person, "benunit": benunit, "household": household}, + # follows it. On more than one clone the evidence records the fold. + lineage_tables = {"person": person, "benunit": benunit, "household": household} + fold = fold_geographic_clones( + lineage_tables, "benunit", training, weights, @@ -336,6 +346,10 @@ def redraw_spi_reported_uc( **{UC_REPORTER_TARGET: np.asarray(reported_before)} ), ) + geographic_fold = geographic_fold_receipt( + lineage_tables, "benunit", fold, stage=UC_REPORTER_REDRAW_STAGE_NAME + ) + training, fit_weights = fold.kept, fold.weights if float(fit_weights[training].sum()) <= 0.0: raise ValueError("UC reporter redraw training weights must have positive mass.") if not target.any(): @@ -453,6 +467,7 @@ def redraw_spi_reported_uc( if UC_REPORTER_GATE_CALIBRATION != UK_DEFAULT_GATE_CALIBRATION else None ), + geographic_fold=geographic_fold, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py index ad33f2f859..16b022d2ea 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_lisa.py @@ -70,6 +70,7 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.hmrc_isa_statistics import ( HMRC_ISA_BUILD_TAX_YEAR, HMRCISAStatistics, @@ -1639,6 +1640,9 @@ def __call__(self, frame: Frame) -> Frame: donor.person[[BALANCE_COLUMN]], columns=UK_WAS_LISA_SUPPORT_CLIP_COLUMNS, stage=UK_WAS_LISA_STAGE_NAME, + clone_count=geographic_clone_count( + household_table, stage=UK_WAS_LISA_STAGE_NAME + ), ) capped, coherence = cap_lifetime_isa_to_financial_wealth( person_household_ids=person_household_ids, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py index d3f757b094..d580897f55 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py @@ -19,6 +19,7 @@ ) from microcosm.build.uk_runtime.frs_brma import _benunit_household_map from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_NET_INCOME_PREDICTOR, UK_HBAI_NET_INCOME_VARIABLE, @@ -477,7 +478,13 @@ def __call__(self, frame: Frame) -> Frame: ) self.last_fit_weight_records = imputation.fit_weight_records capped, capped_rows = cap_derived_totals_to_donor_range(imputation.draws, donor) - clip_result = support_clip_to_donor(capped, donor) + clip_result = support_clip_to_donor( + capped, + donor, + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_WAS_WEALTH_STAGE_NAME + ), + ) household_draws, appended_rows = write_share_split_and_secured_debt( clip_result.clipped ) @@ -1615,15 +1622,20 @@ def values(table: pd.DataFrame, column: str) -> np.ndarray: def support_clip_to_donor( - draws: pd.DataFrame, donor: pd.DataFrame + draws: pd.DataFrame, donor: pd.DataFrame, *, clone_count: int = 1 ) -> UKSupportClipResult: - """Clip output draws to donor-realized support.""" + """Clip output draws to donor-realized support. + + ``clone_count`` is the frame's geographic clone count, which the receipt + carries for the support-clip check (microcosm#931). + """ return support_clip_to_donor_with_receipt( draws, donor, columns=UK_WAS_WEALTH_OUTPUT_COLUMNS, stage=UK_WAS_WEALTH_STAGE_NAME, + clone_count=clone_count, ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_apprentice_draw.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_apprentice_draw.py index b065ed17d9..5697abfff7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_apprentice_draw.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_apprentice_draw.py @@ -230,6 +230,101 @@ def test_receipt_withholds_a_population_on_fewer_than_three_households() -> None assert receipt[key] == "<3", key +def test_receipt_counts_its_support_in_geographic_clone_zero() -> None: + """Each geographic clone flags its own employees, so the receipt counts + its support in clone 0's people, as one clone counts it, and withholds + the figures clone 0 carries on fewer than three households; without the + clone flags every person counts, as before (microcosm#931).""" + + from microcosm.build.uk_runtime.receipt_counts import support_band + + base = _small([22] * 20, ["FT_EMPLOYED"] * 20) + clones = [] + for k in range(3): + clone = base.copy() + clone["person_id"] = base["person_id"] + k * 10**9 + clone["person_household_id"] = base["person_household_id"] + k * 10**9 + clones.append(clone) + person = pd.concat(clones, ignore_index=True) + sources = np.tile(base["person_household_id"].to_numpy(), 3) + clone_zero = np.repeat([True, False, False], len(base)) + weights = np.full(len(person), 1 / 3) + + draw = derive_is_apprentice( + person, + rate=0.05, + person_weights=weights, + person_sources=sources, + person_clone_zero=clone_zero, + ) + in_clone_zero = len(set(sources[draw.flags & clone_zero])) + over_clones = len(set(sources[draw.flags])) + # The premise: clone 0 flags fewer than three households, every clone + # together three or more. + assert support_band(in_clone_zero) == "<3" + assert support_band(over_clones) != "<3" + assert draw.receipt["flagged_support"] == "<3" + assert draw.receipt["realized_mass"] == "<3" + assert draw.receipt["population_support"] == support_band(20) + every = derive_is_apprentice( + person, rate=0.05, person_weights=weights, person_sources=sources + ) + assert every.receipt["flagged_support"] == support_band(over_clones) + assert every.receipt["realized_mass"] != "<3" + np.testing.assert_array_equal(every.flags, draw.flags) + with pytest.raises(ValueError, match="one flag per person"): + derive_is_apprentice( + person, + rate=0.05, + person_sources=sources, + person_clone_zero=clone_zero[:-1], + ) + + +def test_receipt_withholds_what_other_clones_flag_on_one_or_two_households() -> None: + """The masses are taken over every clone, so a mass one or two source + households carry in the other clones is withheld though clone 0 flags no + one, and the printed support stays clone 0's (microcosm#931).""" + + base = _small([22] * 20, ["FT_EMPLOYED"] * 20).assign(employment_income=20_000.0) + clones = [] + for k in range(3): + clone = base.copy() + clone["person_id"] = base["person_id"] + k * 10**9 + clone["person_household_id"] = base["person_household_id"] + k * 10**9 + if k == 0: + # Clone 0 has no employee, so it flags no one. + clone["employment_status"] = "STUDENT" + clones.append(clone) + person = pd.concat(clones, ignore_index=True) + sources = np.tile(base["person_household_id"].to_numpy(), 3) + clone_zero = np.repeat([True, False, False], len(base)) + weights = np.ones(len(person)) + + draw = derive_is_apprentice( + person, + rate=1 / len(person), + person_weights=weights, + relief_schedule={ + "rate": 0.15, + "secondary_threshold": 5_000.0, + "apprentice_upper_secondary_threshold": 50_270.0, + }, + person_sources=sources, + person_clone_zero=clone_zero, + ) + # The premise: clone 0 flags no one, the other clones one household. + assert not draw.flags[clone_zero].any() + assert len(set(sources[draw.flags])) == 1 + assert draw.receipt["flagged_support"] == 0 + assert draw.receipt["eligible_support"] == 0 + for key in ("realized_mass", "realization_deviation", "implied_relief"): + assert draw.receipt[key] == "<3", key + # Clone 0's population and the population over every clone are the same + # twenty households, so the population's figures are given. + assert draw.receipt["population_mass"] == float(len(person)) + + def test_refuses_a_frame_without_employment_status() -> None: person = pd.DataFrame({"person_id": [1], "person_household_id": [1], "age": [22]}) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py index 5674dee327..23f55d8da2 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_battery_bindings.py @@ -2888,3 +2888,145 @@ def test_an_exclusion_key_that_names_no_build_is_refused(self) -> None: outcome = {o.entry.id: o for o in battery.outcomes}["uk_input_mass_parity"] assert outcome.status is GateStatus.FAILED assert "@" in " ".join(outcome.result.failures) + + +# The band donor check reads the phase's frame (microcosm#931). A capped +# seating's receipt gives no exact count, so the check recomputes the seats +# from the transferred phase's frame, which the stage-health binding hands it +# for that check alone. Every frame below is synthetic. + +_BAND_DONOR_CHECK = "spi_income_band_donor_support" +_BAND_DONOR_BOUNDS = (200_000, 500_000, 1_000_000, 2_000_000) + + +def _band_donor_entry(uk_gates): + (entry,) = [ + entry + for entry in uk_gates.gates + if entry.gate == "stage_health" + and entry.parameters["check"] == _BAND_DONOR_CHECK + ] + return entry + + +def _band_donor_frame(*, carrier_on_first_seat: bool = True): + """Two seats per declared band, each with one carrier and one other + member, beside two incumbent households; ids in the band donors' + copy-id ordinal, one clone.""" + + seats = [ + (2 * 10**11 + 100 * position + seat + 21, float(band)) + for position, band in enumerate(_BAND_DONOR_BOUNDS) + for seat in range(2) + ] + ids = [11, 12, *(seat for seat, _ in seats)] + bands = [0.0, 0.0, *(band for _, band in seats)] + donor = [False, False, *([True] * len(seats))] + household = pd.DataFrame( + { + "household_id": np.asarray(ids, dtype=np.int64), + "household_weight": np.full(len(ids), 10.0), + "household_is_spi_income_band_donor": donor, + "spi_income_band_donor_lower_bound": bands, + "household_clone_index": np.zeros(len(ids), dtype=np.int64), + } + ) + carriers = [ + flag and (carrier_on_first_seat or household_id != seats[0][0]) + for household_id, flag in zip(ids, donor, strict=True) + ] + person = pd.DataFrame( + { + "person_id": np.r_[ + np.asarray(ids, dtype=np.int64) * 10, + np.asarray(ids, dtype=np.int64) * 10 + 1, + ], + "person_household_id": np.r_[ids, ids].astype(np.int64), + "person_benunit_id": np.r_[ids, ids].astype(np.int64), + "person_is_spi_income_band_carrier": [*carriers, *([False] * len(ids))], + } + ) + benunit = pd.DataFrame({"benunit_id": np.asarray(ids, dtype=np.int64)}) + return uk_national_frame( + person=person, benunit=benunit, household=household, time_period="2024" + ) + + +def _capped_band_donor_receipt() -> dict[str, object]: + return { + "stage": "spi_income_band_donors", + "minimum_donors_per_band": 120, + "maximum_donor_weight": 250.0, + "maximum_reallocation_share": 0.5, + "clone_count": 1, + "seating_scale": "capped", + "mass_scale": 1.0, + "sample_fraction": 0.01, + "donor_count": 10, + "reallocated_mass": "<3", + "funding_within_share": True, + "funding_matches_bands": True, + "mass": {"old_total": 1_000.0, "new_total": 1_000.0}, + "bands": [ + {"lower_bound": band, "donor_weight": 50.0, "donor_households": "<3"} + for band in _BAND_DONOR_BOUNDS + ], + "funding": [{"stratum": "LONDON", "donor_households": 10}], + } + + +def test_the_stage_health_binding_reads_the_frame_for_the_band_donor_check_only( + uk_gates, +) -> None: + binding = UK_GATE_REGISTRY["stage_health"] + entries = [entry for entry in uk_gates.gates if entry.gate == "stage_health"] + assert any(entry.parameters["check"] != _BAND_DONOR_CHECK for entry in entries) + for entry in entries: + assert binding.requires_frame(entry.parameters) is ( + entry.parameters["check"] == _BAND_DONOR_CHECK + ), entry.id + # The check runs at the spine's transferred boundary, whose battery hands + # every gate the phase's frame. + assert _band_donor_entry(uk_gates).phase == "transferred" + + +def test_the_band_donor_check_recomputes_a_capped_seating_from_the_phase_frame( + uk_gates, +) -> None: + entry = _band_donor_entry(uk_gates) + gates = GatesManifest( + country=uk_gates.country, + version=uk_gates.version, + policy=f"{uk_gates.policy}; band donor binding test", + phases=uk_gates.phases, + gates=(entry,), + ) + artifacts = { + "stage_evidence": {"spi_income_band_donors": _capped_band_donor_receipt()} + } + + def outcome(frame): + report = evaluate_phase( + gates, + "transferred", + EvidenceContext(frame=frame, artifacts=artifacts), + registry=UK_GATE_REGISTRY, + ) + (only,) = report.outcomes + return only + + # Without the phase's frame the check is evidence_absent, never a verdict + # on the receipt alone. + absent = outcome(None) + assert absent.status is GateStatus.EVIDENCE_ABSENT + assert absent.reason == "missing evidence: frame" + # Handed the frame, it recomputes the seats there. + passed = outcome(_band_donor_frame()) + assert passed.status is GateStatus.PASSED, passed.result.failures + assert passed.result.details["seating_scale"] == "capped" + failed = outcome(_band_donor_frame(carrier_on_first_seat=False)) + assert failed.status is GateStatus.FAILED + assert failed.result.failures == ( + "spi_income_band_donors: band from 200000's seats do not each hold one " + "carrier.", + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py index 1358bc1ee9..298cec9316 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_cgt_support.py @@ -841,48 +841,52 @@ def test_geographic_clone_count_is_one_more_than_the_largest_index(): assert geographic_clone_count(empty) == 1 +_CLONE_COUNT_RULE = ( + "A spine counts K geographic clones as the 'household_clone_index' values " + "0 to K - 1 (microcosm#931)." +) + + @pytest.mark.parametrize( "values,message", [ ( np.asarray([0, 2, 2, 0], dtype="int64"), - "1 of the 3 indices from 0 to the largest are carried by no household; " - "the CGT support split counts K geographic clones as the " - "'household_clone_index' values 0 to K - 1 (microcosm#931).", + "An index from 0 to the largest is carried by no household. " + + _CLONE_COUNT_RULE, ), ( np.asarray([0, 3], dtype="int64"), - "2 of the 4 indices from 0 to the largest are carried by no household; " - "the CGT support split counts K geographic clones as the " - "'household_clone_index' values 0 to K - 1 (microcosm#931).", + "An index from 0 to the largest is carried by no household. " + + _CLONE_COUNT_RULE, ), ( np.asarray([0, -1, 1, -2], dtype="int64"), - "2 household(s) carry a negative 'household_clone_index'; the CGT " - "support split counts K geographic clones as the " - "'household_clone_index' values 0 to K - 1 (microcosm#931).", + "A household carries a negative 'household_clone_index'. " + + _CLONE_COUNT_RULE, ), ( pd.array([0, None, 1], dtype="Int64"), - "1 household(s) carry no 'household_clone_index'; the CGT support " - "split counts K geographic clones as the 'household_clone_index' " - "values 0 to K - 1 (microcosm#931).", + "A household carries no 'household_clone_index'. " + _CLONE_COUNT_RULE, ), ( np.asarray([0.0, 1.0]), - "The CGT support split counts geographic clones on an integer " - "'household_clone_index' column; it has dtype float64.", + "'household_clone_index' has dtype float64, not an integer one. " + + _CLONE_COUNT_RULE, ), ( np.asarray([False, True]), - "The CGT support split counts geographic clones on an integer " - "'household_clone_index' column; it has dtype bool.", + "'household_clone_index' has dtype bool, not an integer one. " + + _CLONE_COUNT_RULE, ), ], ) -def test_geographic_clone_count_refuses_indices_other_than_0_to_k_by_count( +def test_geographic_clone_count_refuses_indices_other_than_0_to_k_without_counts( values, message: str ): + """Each refusal is a fixed sentence: no count of households, no id, and + the calling stage where one is named (microcosm#931).""" + household = pd.DataFrame( { "household_id": 1_000_003 + np.arange(len(values)), @@ -895,6 +899,9 @@ def test_geographic_clone_count_refuses_indices_other_than_0_to_k_by_count( assert str(caught.value) == message assert "1000003" not in str(caught.value) + with pytest.raises(ValueError) as named: + geographic_clone_count(household, stage="cgt_support_split") + assert str(named.value) == message.replace("A spine", "cgt_support_split") # --- the split ---------------------------------------------------------------- @@ -2062,7 +2069,7 @@ def counts_by_clone(clone_count: int): assert three_selected == 4 -def test_the_transform_refuses_a_gap_in_the_clone_indices_by_count(): +def test_the_transform_refuses_a_gap_in_the_clone_indices_naming_its_stage(): frame = clone_first_frame(_scenario_frame(), 2) household = frame.table("household").copy() household.loc[household["household_clone_index"] == 1, "household_clone_index"] = 2 @@ -2082,10 +2089,10 @@ def test_the_transform_refuses_a_gap_in_the_clone_indices_by_count(): with pytest.raises(ValueError) as caught: transform(gapped) - # A count and the rule, never a household. + # A fixed sentence naming the stage and the rule: no count, no household. assert str(caught.value) == ( - "1 of the 3 indices from 0 to the largest are carried by no household; " - "the CGT support split counts K geographic clones as the " + "An index from 0 to the largest is carried by no household. " + "cgt_support_split counts K geographic clones as the " "'household_clone_index' values 0 to K - 1 (microcosm#931)." ) assert transform.last_result is None diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py index 07db7b1f71..c496756a58 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_copy_id_table.py @@ -725,7 +725,12 @@ def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: #: donors, which mint through the support channel's clone helper. The #: geographic expansion is the one reader among the clone-first steps: it #: mints the clones and exports their limit, and the geography identity reads -#: clone paths from the lineage columns, never from ids. +#: clone paths from the lineage columns, never from ids. The stage-health +#: gates decode them: the band donor check reads a capped seating's seats as +#: the donor rows whose ids hold no ordinal in a field minted after the band +#: donors, and the regional home values count the sources at the value cap in +#: the rows whose ids hold no geographic_support ordinal, clone 0's +#: (microcosm#931). _COPY_ID_MODULES = frozenset( { *( @@ -738,8 +743,10 @@ def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: "geographic_expand", "graph_kernels", "high_value_home_support", + "regional_home_values", "spi_band_donors", "spi_support", + "stage_health", ) ), "tools/verify_uk_identity_stability.py", @@ -748,6 +755,14 @@ def test_the_executor_refuses_an_incumbent_id_at_its_radix_by_count() -> None: _MINTS_THROUGH_SPI_SUPPORT = frozenset( {"packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py"} ) +#: Modules that name the copy-id table only to hash it, each with why; they +#: mint, bound and decode no id. +_HASHES_THE_COPY_ID_TABLE = { + "packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py": ( + "the transferred spine gate's own code hashes it, since only that " + "phase's band donor check decodes ids through it (microcosm#931)" + ), +} def _uk_python_sources() -> dict[str, str]: @@ -851,7 +866,8 @@ def test_only_the_declared_modules_compute_ten_to_an_ids_digit_count() -> None: def test_no_module_that_mints_or_decodes_copy_ids_names_the_retired_rule() -> None: """Each module that mints, bounds or decodes spine copy ids reads the copy-id table, the identity tool among them, and none names the rule the - table replaced; the readers of the table are exactly these modules.""" + table replaced; the readers of the table are exactly these modules and + the ones that name it only to hash it.""" names = { path: _source_names(source) for path, source in _uk_python_sources().items() @@ -860,7 +876,10 @@ def test_no_module_that_mints_or_decodes_copy_ids_names_the_retired_rule() -> No readers.discard( "packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py" ) - assert readers == _COPY_ID_MODULES - _MINTS_THROUGH_SPI_SUPPORT + assert readers == (_COPY_ID_MODULES - _MINTS_THROUGH_SPI_SUPPORT) | set( + _HASHES_THE_COPY_ID_TABLE + ) + assert not _COPY_ID_MODULES & set(_HASHES_THE_COPY_ID_TABLE) for path in sorted(_COPY_ID_MODULES): assert _RETIRED_RULE not in names[path], path assert not _COPY_ID_MODULES & set(_DENSE_RULE_MODULES) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_services.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_services.py index 21f78b2baf..f6f10432b5 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_services.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_services.py @@ -368,6 +368,83 @@ def _receipt_stage(name: str, fit_kind: str): ) +def test_the_support_clip_receipt_gives_the_frames_clone_count( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Every geographic clone's draws are clipped, so the clip receipt gives + the frame's clone count from two clones up, for the support-clip check to + multiply its row allowances by; a one-clone receipt keeps its form + (microcosm#931).""" + + import numpy as np + + from microcosm.build.uk_runtime import etb_services as module + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + household_columns = list(module.UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS) + clipped = household_columns[:3] + donor = pd.DataFrame({column: [1.0, 2.0] for column in clipped}) + monkeypatch.setattr(module, "assert_rules_engine_country", lambda *_: None) + monkeypatch.setattr( + module, + "etb_services_configuration", + lambda _stage: { + "year": 2024, + "weeks_in_year": 52, + "rail_fare_index": 1.0, + "nhs_budget": 1.0, + }, + ) + monkeypatch.setattr(module, "etb_donor_uprating", lambda _stage: ({}, {})) + monkeypatch.setattr(module, "clean_etb_services_table", lambda raw, **_: raw) + monkeypatch.setattr(module, "recipient_predictors", lambda *_: pd.DataFrame()) + monkeypatch.setattr(module, "_qrf_seed", lambda _stage: 0) + + def impute(*_args, record_ids, **_kwargs): + # Every household draws above the donor's range. + columns = [*household_columns, "rail_subsidy_spending"] + return ( + pd.DataFrame({column: np.full(len(record_ids), 3.0) for column in columns}), + (module.FitWeightRecord("etb_services:test", "explicit"),), + ) + + monkeypatch.setattr(module, "impute_etb_services", impute) + monkeypatch.setattr(module, "support_clip_exempt", lambda _stage: ()) + monkeypatch.setattr(module, "etb_bus_support_pricing", lambda _s, d, **_: (d, {})) + monkeypatch.setattr( + module, + "allocate_nhs_by_age_gender", + lambda person, **_: ( + pd.DataFrame( + { + column: np.zeros(len(person)) + for column in module.UK_NHS_OUTPUT_COLUMNS + } + ), + {}, + ), + ) + + def clip(frame) -> dict: + transform = module.UKETBServicesStageTransform( + stage=_receipt_stage("etb_services", "fit_weighted_qrf_chain"), + engine=object(), + donor=donor, + ) + transform(frame) + return transform.checkpoint_metadata()["evidence"]["support_clip"] + + one = clip(_receipt_frame()) + assert set(one) == {"columns"} + assert {receipt["clipped_high_rows"] for receipt in one["columns"].values()} == {2} + cloned = clip(clone_first_frame(_receipt_frame(), 3)) + assert cloned["clone_count"] == 3 + assert set(cloned["columns"]) == set(clipped) + assert {receipt["clipped_high_rows"] for receipt in cloned["columns"].values()} == { + 6 + } + + def test_stage_transform_records_its_mass_conservation_receipt( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_vat.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_vat.py index bd7dda8ba4..bf83cbcb4b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_vat.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_vat.py @@ -334,6 +334,53 @@ def _receipt_stage(name: str, fit_kind: str): ) +def test_the_support_clip_receipt_gives_the_frames_clone_count( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Every geographic clone's draws are clipped, so the clip receipt gives + the frame's clone count from two clones up, for the support-clip check to + multiply its row allowances by; a one-clone receipt keeps its form + (microcosm#931).""" + + from microcosm.build.uk_runtime import etb_vat as module + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + donor = pd.DataFrame({"full_rate_vat_expenditure_rate": [0.1, 0.2]}) + monkeypatch.setattr(module, "assert_rules_engine_country", lambda *_: None) + monkeypatch.setattr(module, "etb_vat_configuration", lambda _stage: {}) + monkeypatch.setattr(module, "clean_etb_vat_table", lambda raw, **_: donor) + monkeypatch.setattr(module, "recipient_predictors", lambda *_: pd.DataFrame()) + + def impute(*_args, household_ids, **_kwargs): + # Each pair of households draws one rate below the donor's range and + # one above it. + rates = np.resize([0.05, 0.25], len(household_ids)) + return ( + pd.DataFrame({"full_rate_vat_expenditure_rate": rates}), + module.FitWeightRecord("etb_vat:test", "explicit"), + ) + + monkeypatch.setattr(module, "impute_etb_vat", impute) + + def clip(frame) -> dict: + transform = module.UKETBVATStageTransform( + stage=_receipt_stage("etb_vat", "fit_weighted_qrf"), + engine=object(), + donor=donor, + ) + transform(frame) + return transform.checkpoint_metadata()["evidence"]["support_clip"] + + one = clip(_receipt_frame()) + assert set(one) == {"columns"} + rate = one["columns"]["full_rate_vat_expenditure_rate"] + assert (rate["clipped_low_rows"], rate["clipped_high_rows"]) == (1, 1) + cloned = clip(clone_first_frame(_receipt_frame(), 3)) + assert cloned["clone_count"] == 3 + rate = cloned["columns"]["full_rate_vat_expenditure_rate"] + assert (rate["clipped_low_rows"], rate["clipped_high_rows"]) == (3, 3) + + def test_stage_transform_records_its_mass_conservation_receipt( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_clone_collapse.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_clone_collapse.py index b661f85323..462e8dc804 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_clone_collapse.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_geographic_clone_collapse.py @@ -4,6 +4,7 @@ import ast import inspect +import json from pathlib import Path import numpy as np @@ -15,12 +16,14 @@ UK_COPY_LINEAGE, collapse_geographic_clones, fold_geographic_clones, + geographic_fold_receipt, ) from microcosm.build.uk_runtime.national_frame import ( UK_NATIONAL_SCHEMA, uk_national_frame, ) from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column +from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame from test_support.microcosm_build.uk_geographic_clones import with_geographic_clones @@ -199,6 +202,165 @@ def test_a_variant_clone_trains_on_its_own_row_in_reverse_row_order() -> None: assert fold.weights[fold.kept & ~record].tolist() == [1.0] * 3 +def _survey_frame(n: int): + """``n`` survey households of one person in one benefit unit, no copies.""" + + ids = np.arange(1, n + 1, dtype="int64") + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": ids * 10, + "person_household_id": ids, + "person_benunit_id": ids * 10, + } + ), + benunit=pd.DataFrame({"benunit_id": ids * 10}), + household=pd.DataFrame({"household_id": ids, "source_household_id": ids}), + household_weights=np.full(n, 3.0), + time_period="2024", + ) + + +def _fold_receipt(frame, *, varied: int = 0, weightless: tuple[int, ...] = ()): + """The receipt of a fit over every person of ``frame``. + + The clone-1 rows of the survey households whose source id is at most + ``varied`` differ from their other clones on what the fit reads, and the + survey households in ``weightless`` carry no weight. + """ + + tables = _tables(frame) + person, household = tables["person"], tables["household"] + by_household = household.set_index("household_id") + members = person["person_household_id"] + column = ladder_clone_index_column("household") + clone = ( + members.map(by_household[column]).to_numpy() + if column in by_household + else np.zeros(len(person), dtype=np.int64) + ) + source = members.map(by_household["source_household_id"]).to_numpy() + household_weights = pd.Series( + frame.weights_for("household").values, index=by_household.index + ) + weights = np.where( + np.isin(source, weightless), + 0.0, + members.map(household_weights).to_numpy(dtype=float), + ) + fold = fold_geographic_clones( + tables, + "person", + np.ones(len(person), dtype=bool), + weights, + inputs=pd.DataFrame({"value": ((clone == 1) & (source <= varied)) * 1.0}), + ) + return geographic_fold_receipt(tables, "person", fold, stage="The test fit") + + +def test_the_fold_receipt_is_absent_at_one_clone_and_rounded_at_three() -> None: + """A fit's evidence records its geographic fold only on more than one + clone, so a one-clone frame's receipts keep their bytes. Its counts are + true source households of positive weight, rounded as every receipt count + is (microcosm#931).""" + + frame = _survey_frame(25) + assert _fold_receipt(frame) is None + assert _fold_receipt(clone_first_frame(frame, 1)) is None + three = clone_first_frame(frame, 3) + # 25 sources read 30; none varies, so the share reads as its count does. + assert _fold_receipt(three) == { + "clone_count": 3, + "records": 30, + "varied_records": 0, + "varied_share": 0, + } + # A source of no weight carries nothing: 24 sources read 20. + assert _fold_receipt(three, weightless=(25,))["records"] == 20 + # A household's copies are records of their own in the fit, but its + # counts are of sources: one survey household reads "<3" however many + # of its copies vary. + copies = clone_first_frame(_copies_frame(), 3) + assert _fold_receipt(copies, varied=1) == { + "clone_count": 3, + "records": "<3", + "varied_records": "<3", + "varied_share": "<3", + } + + +def test_one_and_two_varied_sources_give_the_same_receipt_bytes() -> None: + """Under three varied source households the count and the share both + read "<3", so frames whose records vary in one source and in two give + the same bytes, as frames varying in three and in nine do ("3-9"). The + share is a floored hundredth only from ten varied sources up + (microcosm#931).""" + + three = clone_first_frame(_survey_frame(25), 3) + + def receipt_bytes(varied: int) -> bytes: + receipt = _fold_receipt(three, varied=varied) + return json.dumps(receipt, sort_keys=True).encode() + + assert receipt_bytes(1) == receipt_bytes(2) + assert _fold_receipt(three, varied=1) == { + "clone_count": 3, + "records": 30, + "varied_records": "<3", + "varied_share": "<3", + } + assert receipt_bytes(3) == receipt_bytes(9) + assert _fold_receipt(three, varied=9)["varied_share"] == "3-9" + # Ten of 25 sources vary, the fewest a share is given at: the count reads + # 10 and the share 0.4. + assert _fold_receipt(three, varied=10) == { + "clone_count": 3, + "records": 30, + "varied_records": 10, + "varied_share": 0.4, + } + # Twelve of 25 sources vary: the count reads 10 and the share 0.48. + assert _fold_receipt(three, varied=12) == { + "clone_count": 3, + "records": 30, + "varied_records": 10, + "varied_share": 0.48, + } + + +def test_the_share_is_withheld_where_the_sources_that_do_not_vary_are_few() -> None: + """The share is given only where the selected sources that do not vary + are none or ten or more as well, so it never places them in a band: a + frame where every source but one varies gives the bytes of one where + every source but two does, and every source but three those of every + source but nine. Where every source varies the share is 1 + (microcosm#931).""" + + three = clone_first_frame(_survey_frame(25), 3) + + def receipt_bytes(varied: int) -> bytes: + receipt = _fold_receipt(three, varied=varied) + return json.dumps(receipt, sort_keys=True).encode() + + assert receipt_bytes(24) == receipt_bytes(23) + assert receipt_bytes(22) == receipt_bytes(16) + # The share falls back to the varied count, rounded as every count is. + assert _fold_receipt(three, varied=24) == { + "clone_count": 3, + "records": 30, + "varied_records": 20, + "varied_share": 20, + } + # Ten sources do not vary: the share is given again. + assert _fold_receipt(three, varied=15)["varied_share"] == 0.6 + assert _fold_receipt(three, varied=25) == { + "clone_count": 3, + "records": 30, + "varied_records": 30, + "varied_share": 1.0, + } + + #: The stages whose fits fold clones, through either UK entry, and the module #: each one's call is in: the SPI income stage runs its forests from #: spi_income. @@ -224,6 +386,23 @@ def _calls_fold(path: Path) -> bool: ) +def test_every_collapsing_stage_records_its_fold() -> None: + """Every module whose fit folds geographic clones also builds the fold's + receipt (``geographic_fold_receipt``), which its stage's evidence gives + on a spine of more than one clone; so a new fold site cannot leave its + fold out of its evidence (microcosm#931).""" + + runtime = Path(inspect.getfile(graph_kernels)).parent + for module in sorted(set(_COLLAPSING_STAGES.values())): + tree = ast.parse((runtime / f"{module}.py").read_text(encoding="utf-8")) + assert any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "geographic_fold_receipt" + for node in ast.walk(tree) + ), module + + def test_every_collapsing_stage_sees_the_lineage_live_at_it() -> None: """A finite surface prunes what it does not declare, and a step whose columns are pruned reads as one that has not run: its copies would fold diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 19b2233cfd..6036f7c92a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -344,12 +344,14 @@ def changed_helper_bytes(path: Path, helper=helper_path, read=original): #: channel's clone helper (``spi_support``), which reads the copy-id table; #: every other minting stage's module reads the table itself (microcosm#931). _SPI_COPY_ID_STAGES = frozenset({"spi_support_channel", "spi_income_band_donors"}) -#: Stages that bind ``spi_support`` or ``frs_spine``, which read the copy-id -#: table, for names that never mint, decode or bound a copy id: constants, -#: the support channel's column names and source fill, and frs_spine's tab -#: reader, id normaliser, interview dates and self-employment rule. They need -#: not bind the table; the helper rule below holds that none of them reaches -#: the table's code another way. +#: Stages that bind ``spi_support``, ``frs_spine`` or ``stage_health``, which +#: read the copy-id table, for names that never mint, decode or bound a copy +#: id: constants, the support channel's column names and source fill, +#: frs_spine's tab reader, id normaliser, interview dates and self-employment +#: rule, and stage_health's latent-attribute tolerance (the band donor check +#: reads the table since microcosm#931). They need not bind the table; the +#: helper rule below holds that none of them reaches the table's code another +#: way. _COPY_ID_TABLE_NAME_ONLY_STAGES = frozenset( { "company_cars", @@ -368,15 +370,20 @@ def changed_helper_bytes(path: Path, helper=helper_path, read=original): "property_components", "spi_benefit_coherence", "uc_capital_coherence", + "uc_deduction_attributes", + "uc_take_up", "was_lisa", "was_wealth", } ) -#: The functions those stages may import from the two helpers. +#: The helpers those stages bind for table-free names, and the functions +#: they may import from them. +_COPY_ID_TABLE_READING_HELPERS = frozenset({"spi_support", "frs_spine", "stage_health"}) _COPY_ID_TABLE_FREE_HELPER_FUNCTIONS = frozenset( { "fill_support_channel_from_source", "frs_interview_dates", + "latent_attribute_tolerance", "normalize_ids", "read_pinned_tab", "support_channel_column", @@ -433,7 +440,7 @@ def test_every_stage_that_mints_or_bounds_copy_ids_binds_the_copy_id_table() -> assert "copy_id_table" not in _imported_module_names(module), stage functions = { name - for name in _names_imported_from(module, {"spi_support", "frs_spine"}) + for name in _names_imported_from(module, _COPY_ID_TABLE_READING_HELPERS) if not name.isupper() } assert functions <= _COPY_ID_TABLE_FREE_HELPER_FUNCTIONS, stage @@ -469,10 +476,11 @@ def changed_helper_bytes(path: Path) -> bytes: return content monkeypatch.setattr(Path, "read_bytes", changed_helper_bytes) - # Eleven since the geographic clones mint their ids in the table's - # geographic_support field (microcosm#931). - assert "geographic_expand" in stages - assert len(affected) == 11 + # Twelve since the geographic clones mint their ids in the table's + # geographic_support field and regional_home_values reads clone 0 from + # it (microcosm#931). + assert {"geographic_expand", "regional_home_values"} <= set(stages) + assert len(affected) == 12 for ref in affected: assert registry.implementation_hash(ref) != before[ref], ref for ref in controls: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py index d621a5bcb2..2939429d7c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_evidence.py @@ -360,6 +360,22 @@ def test_spine_inventory_is_roster_derived_and_gates_bind_checkpoint_versions( "spine.gates.assembled", "was_wealth", } + # The band donor check recomputes a capped seating's seats from the + # transferred phase's frame, so the gate reads the cells it decodes; the + # executor adds the ids and memberships itself (microcosm#931). + from microcosm.build.uk_runtime import stage_health + + read = { + (cells.entity, column) + for cells in transferred.inputs + for column in cells.columns + } + assert { + ("household", stage_health._BAND_DONOR_FLAG_COLUMN), + ("household", stage_health._BAND_DONOR_BAND_COLUMN), + ("household", stage_health._BAND_DONOR_CLONE_INDEX_COLUMN), + ("person", stage_health._BAND_CARRIER_FLAG_COLUMN), + } <= read # The gate admits whichever stage follows the BRMA checkpoint in the roster # (the SPI block since #1012 moved it ahead of was_wealth), as main's tool # does with UK_SPINE_ASSEMBLED_FINAL_STAGE. Since the HMRC leaves and the diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_k_sensitive_constants.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_k_sensitive_constants.py new file mode 100644 index 0000000000..2968371341 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_k_sensitive_constants.py @@ -0,0 +1,522 @@ +"""How each count-like constant of the UK spine meets the geographic clone count. + +The clone-first spine gives every FRS household ``K`` geographic clones +(microcosm#931), each at ``1 / K`` of its weight. A threshold on rows then +sees ``K`` times the rows it saw at one clone, while one on distinct true +source households, on survey records or on weight does not. Each constant +below is classified by what it counts and by how the code meets ``K``, and +pinned to the code's value by import, so a change to one fails here until +it is classified again. The gates' count-like maxima are pinned the same +way: a new one in the packaged gates must join the table. +""" + +from __future__ import annotations + +import inspect +import json +import re +from collections.abc import Callable +from dataclasses import dataclass +from importlib.resources import files + +import pytest + +from microcosm.build import stochastic_assignment +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime import ( + aps_occupation, + cgt_support, + company_cars, + frs_person_draws, + frs_take_up, + regional_home_values, + salary_sacrifice, + spi_band_donors, + spi_employment_shell, + spi_rank_transport, + stage_health, + support_clip, + uc_capital_coherence, +) + +#: What a constant counts. +COUNTS = frozenset( + {"source count", "survey records", "rows", "weight", "hyperparameter"} +) +#: How the code meets the clone count, each with what it means. +TREATMENTS = { + "per clone": ( + "declared for one clone: the stage applies it within each clone, and " + "its check holds K times the stage's figure to it" + ), + "invariant": "what it counts does not move with K", + "kept": "it moves with K, and is kept as it is for the reason given", + "scaled by K": ( + "a row allowance the stage-health check multiplies by the receipt's " + "clone_count, which a stage gives only from two clones up" + ), + "clone 0": "counted in geographic clone 0's rows, which do not move with K", + "later": "taken up later in microcosm#931, for the reason given", +} + + +@dataclass(frozen=True) +class KSensitiveConstant: + """One constant: where it lives, its value, what it counts and how the + code meets ``K``. ``read`` returns the code's value now.""" + + name: str + where: str + read: Callable[[], object] + value: object + counts: str + treatment: str + reason: str + #: The packaged gates' (gate id, parameter) pairs that declare it. + gate_parameters: frozenset[tuple[str, str]] = frozenset() + + +def _gates() -> list[dict]: + payload = files("microcosm.build.uk").joinpath("gates.json").read_text() + return json.loads(payload)["gates"] + + +def _declared_parameters(check: str, *keys: str) -> frozenset[tuple[str, str]]: + """The packaged gates of ``check`` that declare each of ``keys``.""" + + return frozenset( + (gate["id"], key) + for gate in _gates() + if (gate.get("parameters") or {}).get("check") == check + for key in keys + if key in gate["parameters"] + ) + + +def _company_car_donor_households() -> int: + stage = load_country_spec("uk").sources.stage_map()["company_cars"] + return company_cars.CompanyCarsSpec.from_stage(stage).minimum_donor_households + + +def _source_names(function: Callable[..., object], *names: str) -> bool: + source = inspect.getsource(function) + return all(name in source for name in names) + + +_SUPPORT_CLIP_STAGES = ( + "was_wealth", + "was_lisa", + "nts_bus_travel", + "lcfs_consumption", + "etb_vat", + "etb_services", +) +_CLIP_ALLOWANCES = ( + "max_clipped_low_rows_by_column", + "max_clipped_high_rows_by_column", +) +_PROPERTY_MAXIMA = ( + "maximum_rows_receipts_below_profit", + "maximum_rows_with_profit_above_band", +) + +K_SENSITIVE_CONSTANTS: tuple[KSensitiveConstant, ...] = ( + KSensitiveConstant( + name="band donor maximum weight", + where="spi_band_donors.SPI_INCOME_BAND_MAXIMUM_DONOR_WEIGHT", + read=lambda: spi_band_donors.SPI_INCOME_BAND_MAXIMUM_DONOR_WEIGHT, + value=250.0, + counts="weight", + treatment="per clone", + reason=( + "each clone's donors weigh a 1 / K share, and the band donor " + "check holds K times a donor's weight to the maximum" + ), + ), + KSensitiveConstant( + name="CGT support copy maximum weight", + where="cgt_support.CGT_SUPPORT_MAXIMUM_COPY_WEIGHT", + read=lambda: cgt_support.CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + value=60.0, + counts="weight", + treatment="per clone", + reason="a copy weighs at most the maximum divided by K", + ), + KSensitiveConstant( + name="band donor minimum seats", + where="spi_band_donors.SPI_INCOME_BAND_MINIMUM_DONORS", + read=lambda: spi_band_donors.SPI_INCOME_BAND_MINIMUM_DONORS, + value=120, + counts="source count", + treatment="per clone", + reason=( + "every band seats at least this many donors in each clone, and " + "the band donor check requires K times it" + ), + ), + KSensitiveConstant( + name="band donor household share", + where="spi_band_donors.SPI_INCOME_BAND_MAXIMUM_DONOR_HOUSEHOLD_SHARE", + read=lambda: spi_band_donors.SPI_INCOME_BAND_MAXIMUM_DONOR_HOUSEHOLD_SHARE, + value=0.15, + counts="source count", + treatment="invariant", + reason=( + "a share of the eligible base households counted in distinct " + "source households" + ), + ), + KSensitiveConstant( + name="company car donor households", + where=( + "the company_cars stage's minimum_donor_households " + "(company_cars.CompanyCarsSpec)" + ), + read=_company_car_donor_households, + value=3, + counts="source count", + treatment="invariant", + reason="distinct WAS donor households, and donors are never cloned", + ), + KSensitiveConstant( + name="bonus sacrifice donor sources", + where="salary_sacrifice.BONUS_SACRIFICE_SPI_MIN_DONOR_SOURCES", + read=lambda: salary_sacrifice.BONUS_SACRIFICE_SPI_MIN_DONOR_SOURCES, + value=20, + counts="source count", + treatment="invariant", + reason="distinct true source households among a cell's donors", + ), + KSensitiveConstant( + name="UC capital cell donors", + where="uc_capital_coherence.UC_CAPITAL_MINIMUM_CELL_DONORS", + read=lambda: uc_capital_coherence.UC_CAPITAL_MINIMUM_CELL_DONORS, + value=20, + counts="survey records", + treatment="invariant", + reason=( + "a benefit unit's geographic clones count once, whether they fold " + "into one row or donate apart" + ), + ), + KSensitiveConstant( + name="CGT support correction candidates", + where="cgt_support.CGT_SUPPORT_CORRECTION_CANDIDATES", + read=lambda: cgt_support.CGT_SUPPORT_CORRECTION_CANDIDATES, + value=64, + counts="rows", + treatment="kept", + reason=( + "the heaviest whole households tried for the exact-total " + "correction; they cover fewer sources at K clones, and the " + "correction they place is of rounding size" + ), + ), + KSensitiveConstant( + name="SPI transport group minimum rows", + where="spi_rank_transport.SPI_TRANSPORT_GROUP_MINIMUM_ROWS", + read=lambda: spi_rank_transport.SPI_TRANSPORT_GROUP_MINIMUM_ROWS, + value=10, + counts="rows", + treatment="kept", + reason=( + "a statistical floor of a uniformity diagnostic only the receipt " + "reports; at K clones it reports more groups" + ), + ), + KSensitiveConstant( + name="support-clip row allowances", + where=( + "the six support-clip gates' max_clipped_low_rows_by_column and " + "max_clipped_high_rows_by_column (stage_health, support_clip)" + ), + read=lambda: _declared_parameters("support_clip", *_CLIP_ALLOWANCES), + value=frozenset( + (f"uk_stage_{stage}_support", key) + for stage in _SUPPORT_CLIP_STAGES + for key in _CLIP_ALLOWANCES + ), + counts="rows", + treatment="scaled by K", + reason="every clone's draws are clipped", + gate_parameters=frozenset( + (f"uk_stage_{stage}_support", key) + for stage in _SUPPORT_CLIP_STAGES + for key in _CLIP_ALLOWANCES + ), + ), + KSensitiveConstant( + name="property components row maxima", + where=( + "the property components gate's maximum_rows_receipts_below_profit " + "and maximum_rows_with_profit_above_band (stage_health)" + ), + read=lambda: _declared_parameters("property_components", *_PROPERTY_MAXIMA), + value=frozenset( + ("uk_stage_property_components", key) for key in _PROPERTY_MAXIMA + ), + counts="rows", + treatment="scaled by K", + reason="every clone's landlords are walked", + gate_parameters=frozenset( + ("uk_stage_property_components", key) for key in _PROPERTY_MAXIMA + ), + ), + KSensitiveConstant( + name="regional home values sources at the cap", + where=( + "the regional home values gate's maximum_sources_at_cap " + "(stage_health), against the stage's sources_at_cap " + "(regional_home_values)" + ), + read=lambda: _declared_parameters( + "regional_home_values", "maximum_sources_at_cap" + ), + value=frozenset( + {("uk_stage_regional_home_values_ranking", "maximum_sources_at_cap")} + ), + counts="source count", + treatment="clone 0", + reason=( + "each clone ranks on its own draw, so the sources at the cap over " + "every clone grow with K" + ), + gate_parameters=frozenset( + {("uk_stage_regional_home_values_ranking", "maximum_sources_at_cap")} + ), + ), + KSensitiveConstant( + name="relationship grid mismatches", + where=( + "the household composition gate's max_grid_reciprocity_mismatches " + "(stage_health), against frs_relationships' receipt" + ), + read=lambda: _declared_parameters( + "household_composition", "max_grid_reciprocity_mismatches" + ), + value=frozenset( + { + ( + "uk_stage_frs_relationships_composition", + "max_grid_reciprocity_mismatches", + ) + } + ), + counts="survey records", + treatment="invariant", + reason=( + "frs_relationships runs before geographic_expand, so the grids it " + "counts are the survey households' own" + ), + gate_parameters=frozenset( + { + ( + "uk_stage_frs_relationships_composition", + "max_grid_reciprocity_mismatches", + ) + } + ), + ), + KSensitiveConstant( + name="take-up signal under-2 usage population support", + where=( + "frs_take_up.uk_take_up_signal_receipt's population_support of the " + "under-2 usage row" + ), + read=lambda: _source_names( + frs_take_up.uk_take_up_signal_receipt, + "population_support", + "UK_UNDER_TWO_USAGE_SIGNAL_KEY", + ), + value=True, + counts="source count", + treatment="clone 0", + reason=( + "the population is the children under 2 in units each clone draws " + "as claiming the extended hours, so its sources over every clone " + "grow with K, and with them the rows whose share is withheld" + ), + ), + KSensitiveConstant( + name="take-up signal distinct values", + where="frs_take_up.uk_take_up_signal_receipt's distinct_values", + read=lambda: _source_names( + frs_take_up.uk_take_up_signal_receipt, "distinct_values" + ), + value=True, + counts="rows", + treatment="clone 0", + reason=( + "each clone draws its own flags, so a flag constant on one clone " + "can vary over K" + ), + ), + KSensitiveConstant( + name="apprentice flagged support", + where="frs_person_draws.derive_is_apprentice's flagged_support", + read=lambda: _source_names( + frs_person_draws.derive_is_apprentice, "flagged_support" + ), + value=True, + counts="source count", + treatment="clone 0", + reason=( + "each clone flags its own employees, so the flagged sources over " + "every clone grow with K, and with them the figures it withholds" + ), + ), + KSensitiveConstant( + name="occupation leaf minimum", + where=( + "min_samples_leaf of spi_employment_shell's occupation model and " + "aps_occupation's imputation" + ), + read=lambda: ( + spi_employment_shell.UK_SPI_EMPLOYMENT_SHELL_OPERATION["occupation"][ + "boosted" + ]["min_samples_leaf"], + aps_occupation.UK_APS_OCCUPATION_IMPUTATION["boosted"]["min_samples_leaf"], + ), + value=(40, 40), + counts="hyperparameter", + treatment="later", + reason=( + "rows of a fit, where geographic clones fold together unless they " + "differ; it is taken up with the fits' cross-fitted folds by true " + "source, later in #931" + ), + ), + KSensitiveConstant( + name="band donor reallocation share", + where="spi_band_donors.SPI_INCOME_BAND_MAXIMUM_REALLOCATION_SHARE", + read=lambda: spi_band_donors.SPI_INCOME_BAND_MAXIMUM_REALLOCATION_SHARE, + value=0.5, + counts="weight", + treatment="later", + reason=( + "a share of each funding stratum's mass, the stratum now a region " + "within a clone; its common mass scale is measured on a spine of " + "fifteen clones before any change" + ), + ), + KSensitiveConstant( + name="take-up target count", + where="stochastic_assignment.assign_binary_with_anchored_residual", + read=lambda: _source_names( + stochastic_assignment.assign_binary_with_anchored_residual, + "int(rate * int(population.sum()))", + ), + value=True, + counts="rows", + treatment="later", + reason=( + "int(rate x n) rounds once over the rows of every clone; #931 " + "group (c) takes it up with the per-clone take-up counts" + ), + ), + KSensitiveConstant( + name="zero-weight SPI strata rows", + where="the zero-weight strata gate's maximum_zero_weight_rows", + read=lambda: frozenset( + ("uk_zero_weight_strata", "maximum_zero_weight_rows") + for gate in _gates() + if gate["id"] == "uk_zero_weight_strata" + for declaration in gate["parameters"]["declarations"] + if "maximum_zero_weight_rows" in declaration + ), + value=frozenset({("uk_zero_weight_strata", "maximum_zero_weight_rows")}), + counts="rows", + treatment="later", + reason=( + "a terminal gate: the national and dense builds that run it refuse " + "a clone-first spine until #931 group (c)" + ), + gate_parameters=frozenset( + {("uk_zero_weight_strata", "maximum_zero_weight_rows")} + ), + ), +) + + +@pytest.mark.parametrize( + "constant", K_SENSITIVE_CONSTANTS, ids=lambda constant: constant.name +) +def test_each_constant_is_the_codes(constant: KSensitiveConstant) -> None: + assert constant.read() == constant.value, constant.where + + +def test_each_constant_is_classified() -> None: + names = [constant.name for constant in K_SENSITIVE_CONSTANTS] + assert len(set(names)) == len(names) + for constant in K_SENSITIVE_CONSTANTS: + assert constant.counts in COUNTS, constant.name + assert constant.treatment in TREATMENTS, constant.name + assert constant.where and constant.reason, constant.name + # A row threshold is never invariant: it is scaled, counted in clone 0, + # kept for its reason or taken up later. + assert not [ + constant.name + for constant in K_SENSITIVE_CONSTANTS + if constant.counts == "rows" and constant.treatment == "invariant" + ] + + +#: A gate parameter that caps a count: maxima of rows, sources, households, +#: records, donors, cells or mismatches. +_COUNT_MAXIMUM = re.compile( + r"^max(imum)?_(.*_)?" + r"(rows|sources|households|records|donors|cells|mismatches|count)(_.*)?$" +) + + +def _count_maxima(value: object) -> set[str]: + if isinstance(value, dict): + found = {key for key in value if _COUNT_MAXIMUM.match(key)} + for item in value.values(): + found |= _count_maxima(item) + return found + if isinstance(value, list): + return set().union(*(_count_maxima(item) for item in value)) + return set() + + +def test_every_count_maximum_in_the_gates_is_classified() -> None: + """A count-like maximum a packaged gate declares is in the table, so a + new one is classified before a spine of several clones reads it.""" + + declared = { + (gate["id"], key) + for gate in _gates() + for key in _count_maxima(gate.get("parameters") or {}) + } + classified = set().union( + *(constant.gate_parameters for constant in K_SENSITIVE_CONSTANTS) + ) + assert declared == classified + + +def test_the_scaled_and_clone_zero_treatments_are_the_codes() -> None: + """The row allowances the table scales are read through the receipt's + clone count, which the shared clip receipt and the property components + stage give; the counts the table counts in clone 0 are the stages'.""" + + for function in ( + stage_health._support_clip_gate, + stage_health._property_components_gate, + ): + assert _source_names(function, "_receipt_clone_count", "clone_count") + assert _source_names(support_clip.UKSupportClipReceipt.evidence, '"clone_count"') + assert _source_names( + regional_home_values.assign_regional_home_values, + "clone_zero", + "GEOGRAPHIC_COPY_ID_FIELD", + ) + # The regional home values check reads its declared maximum as it stands. + assert not _source_names(stage_health._regional_home_values_gate, "clone_count") + assert _source_names( + frs_take_up.uk_take_up_signal_receipt, "clone_zero", "population_support" + ) + assert _source_names(frs_take_up._measure_take_up_signal, "clone_zero") + assert _source_names( + frs_person_draws.derive_is_apprentice, "person_clone_zero", "flagged_support" + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 76d1f11734..3ee2c0c043 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -1743,6 +1743,31 @@ def materialize(self, frame, variables, period): ) +def test_the_support_clip_receipt_gives_the_frames_clone_count() -> None: + """Every geographic clone's draws are clipped, so the clip receipt gives + the frame's clone count from two clones up, for the support-clip check to + multiply its row allowances by; a one-clone receipt keeps its form + (microcosm#931).""" + + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + def clip(clones: int) -> dict: + synthetic = _synthetic_lcfs_stage() + frame = synthetic.frame + synthetic.transform(frame if clones == 1 else clone_first_frame(frame, clones)) + return synthetic.transform.checkpoint_metadata()["evidence"]["support_clip"] + + one = clip(1) + assert set(one) == {"columns"} + cloned = clip(3) + assert cloned["clone_count"] == 3 + assert set(cloned) == {"columns", "clone_count"} + assert all( + receipt["rows_considered"] == 3 * one["columns"][column]["rows_considered"] + for column, receipt in cloned["columns"].items() + ) + + def test_stage_transform_prices_bus_fares_from_journeys() -> None: """End to end on synthetic inputs: the committed declaration minus the engine. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_nts_bus_travel.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_nts_bus_travel.py index a0ceecd333..cf937e16c7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_nts_bus_travel.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_nts_bus_travel.py @@ -800,6 +800,55 @@ def test_published_facts_read_the_vendored_rows() -> None: ) +def test_the_support_clip_receipt_gives_the_frames_clone_count() -> None: + """Every geographic clone's draws are clipped, so the clip receipt gives + the frame's clone count from two clones up, for the support-clip check to + multiply its row allowances by; a one-clone receipt keeps its form + (microcosm#931).""" + + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + committed = _committed_stage() + stage = dataclasses.replace( + committed, + operations=tuple( + SourceOperationSpec( + kind=op.kind, parameters={**op.parameters, "n_estimators": 4} + ) + if op.kind == IMPUTE_BUS_USE_BAND_KIND + else op + for op in committed.operations + ), + ) + + def clip(clones: int) -> dict: + household, individual, trip, nts_stage, nts_ticket = ( + _synthetic_nts_with_tickets() + ) + frame, rng = _recipient_frame(n=60) + transform = UKNTSBusTravelStageTransform( + stage=stage, + engine=_Engine(rng), + nts_household=household, + nts_individual=individual, + nts_trip=trip, + nts_stage=nts_stage, + nts_ticket=nts_ticket, + ) + transform(frame if clones == 1 else clone_first_frame(frame, clones)) + return transform.checkpoint_metadata()["evidence"]["support_clip"] + + one = clip(1) + assert set(one) == {"columns"} + cloned = clip(3) + assert cloned["clone_count"] == 3 + assert set(cloned) == {"columns", "clone_count"} + assert all( + receipt["rows_considered"] == 3 * one["columns"][column]["rows_considered"] + for column, receipt in cloned["columns"].items() + ) + + def test_stage_transform_end_to_end_on_synthetic_inputs() -> None: committed = _committed_stage() operations = tuple( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py index ee0afe81d6..648866010f 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_components.py @@ -825,3 +825,66 @@ def test_the_stage_adds_frs_costs_back_and_gives_every_landlord_receipts( "Landlords' receipts and finance costs on the source spine: household " "weights pass through unchanged and total household mass is conserved." ) + + def test_the_receipt_gives_the_clone_count_beyond_one_clone(self, tmp_path) -> None: + """The row counts the stage's check holds grow with the geographic + clone count, so the receipt gives the frame's count from two clones + up, for the check to multiply its maxima by (microcosm#931); a + one-clone receipt keeps its form.""" + + from microcosm.build.uk_runtime.property_components import ( + UKPropertyComponentsStageTransform, + ) + from microcosm.build.uk_runtime.spi_income import SPIDonorAgeModel + from test_support.microcosm_build.uk_clone_first_frames import ( + clone_first_frame, + ) + from test_support.microcosm_build.uk_spi_income import _write_donor + + _write_donor(tmp_path / "donor.tab") + frame = self._frame() + person = frame.table("person") + weights = ( + person["person_household_id"] + .map( + pd.Series( + frame.weights_for("household").values, + index=frame.table("household")["household_id"].to_numpy(), + ) + ) + .to_numpy() + ) + + def evidence(frame) -> dict: + transform = UKPropertyComponentsStageTransform( + stage=_stage(), + donor_table=pd.read_csv(tmp_path / "donor.tab", sep="\t"), + age_model=SPIDonorAgeModel( + populations={ + (sex, age): 1.0 + for sex in ("MALE", "FEMALE") + for age in range(91) + }, + state_pension_age=66, + source="flat test populations", + ), + uprating_factors={ + "property_income": 1.0, + "property_finance_costs": 1.0, + }, + facts=_facts( + landlords=float(weights[person["property_income"] > 0].sum()), + receipts=1.65 * float((person["property_income"] * weights).sum()), + ), + ) + transform(frame) + return transform.checkpoint_metadata()["evidence"] + + one = evidence(frame) + assert "clone_count" not in one + indexed = clone_first_frame(frame, 1) + assert "household_clone_index" in indexed.table("household") + assert "clone_count" not in evidence(indexed) + cloned = evidence(clone_first_frame(frame, 3)) + assert cloned["clone_count"] == 3 + assert set(cloned) == {*one, "clone_count"} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_regional_home_values.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_regional_home_values.py index 0e90e47af4..3f0c2756c8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_regional_home_values.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_regional_home_values.py @@ -26,6 +26,7 @@ UK_RELEASE_EXPORT_DROPPED_COLUMNS, uk_national_frame, ) +from microcosm.build.uk_runtime.receipt_counts import support_band from microcosm.frame import Frame, WeightKind from microcosm.graph import ( ContentStore, @@ -41,6 +42,7 @@ write_toy_property_resources, write_toy_supports, ) +from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame from test_support.microcosm_build.uk_full_population_graph import ( SOURCE_VINTAGE, Source, @@ -350,6 +352,194 @@ def test_the_cap_cell_is_banded_on_its_source_households(monkeypatch) -> None: assert _graph_evidence(frame)["sources_at_cap"] == "3-9" +def _cloned_london_owners(monkeypatch, *, n: int = 20, quantile: float = 0.7): + """``n`` London owner households as three geographic clones whose values + differ by clone, as a per-clone draw upstream leaves them, under a cap + the test sets at the London knots' ``quantile``. Returns the frame and + the cap.""" + + parsed, knots = rhv.load_region_knots() + cap = float(np.interp(quantile, parsed.grid, knots["E12000007"])) + monkeypatch.setattr( + rhv, + "load_region_knots", + lambda payload=None: (dataclasses.replace(parsed, cap=cap), knots), + ) + frame = _frame(n=n, seed=3) + household = frame.table("household") + household["region"] = pd.array(["LONDON"] * n, dtype="string") + household["tenure_type"] = pd.array(["OWNED_OUTRIGHT"] * n, dtype="string") + household["main_residence_value"] = np.linspace(300_000.0, 900_000.0, n) + household["property_wealth"] = household["main_residence_value"] + 10_000.0 + household[HIGH_VALUE_SUPPORT_COPY_COLUMN] = False + # The node's shape anchor, the preceding support stage's copy index. + household["high_value_support_copy_index"] = np.zeros(n, dtype="int64") + cloned = clone_first_frame(frame, 3) + household = cloned.table("household") + clone = household["household_clone_index"].to_numpy() + values = household["main_residence_value"].to_numpy().copy() + draw = np.random.default_rng(11) + for k in (1, 2): + rows = np.flatnonzero(clone == k) + values[rows] = draw.permutation(values[rows]) + household["main_residence_value"] = values + household["property_wealth"] = values + 10_000.0 + return cloned, cap + + +def test_the_cap_cell_counts_geographic_clone_zero(monkeypatch) -> None: + """Each geographic clone ranks on its own draw, so the source households + at the cap over every clone grow with the clone count; the receipt counts + clone 0's rows, so the check's one declared maximum holds at any count. + The production node is not given the clone index: the stage reads each + row's clone from its id (microcosm#931).""" + + frame, cap = _cloned_london_owners(monkeypatch) + result = rhv.assign_regional_home_values(frame) + household = result.frame.table("household") + at_cap = household["main_residence_value"].to_numpy() >= cap + sources = household["source_household_id"].to_numpy() + clone_zero = household["household_clone_index"].to_numpy() == 0 + in_clone_zero = support_band(len(set(sources[at_cap & clone_zero]))) + over_clones = support_band(len(set(sources[at_cap]))) + # The premise: the union over the clones prints another band. + assert in_clone_zero != over_clones + assert result.receipt["sources_at_cap"] == in_clone_zero + assert "household_clone_index" not in _graph_input_columns() + assert _graph_evidence(frame)["sources_at_cap"] == in_clone_zero + # Without the clone index the stage reads the same clones from the ids. + unindexed = frame.table("household").drop(columns="household_clone_index") + stripped = uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=unindexed, + time_period="2024", + weight_kind=WeightKind.DESIGN, + household_weights=frame.weights_for("household").values, + ) + assert ( + rhv.assign_regional_home_values(stripped).receipt["sources_at_cap"] + == in_clone_zero + ) + # Clones whose ids were minted another way would all read as clone 0, so + # a frame whose clone index disagrees with its ids is refused. + moved = frame.table("household").copy() + moved["household_clone_index"] = np.roll( + moved["household_clone_index"].to_numpy(), 1 + ) + misread = uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=moved, + time_period="2024", + weight_kind=WeightKind.DESIGN, + household_weights=frame.weights_for("household").values, + ) + with pytest.raises(ValueError, match="disagrees with the frame's") as refused: + rhv.assign_regional_home_values(misread) + assert not any( + character.isdigit() for character in str(refused.value).split("(")[0] + ) + + +def _owners_by_id(ids, sources, owner, copy) -> Frame: + """London households with the given ids, sources, owner flags and support + copy flags, one person and benefit unit each, in id order.""" + + order = np.argsort(np.asarray(ids, dtype="int64"), kind="stable") + ids = np.asarray(ids, dtype="int64")[order] + owner = np.asarray(owner, dtype=bool)[order] + value = np.where(owner, 500_000.0, 0.0) + household = pd.DataFrame( + { + "household_id": ids, + "region": pd.array(["LONDON"] * len(ids), dtype="string"), + "tenure_type": pd.array( + np.where(owner, "OWNED_OUTRIGHT", "RENT_PRIVATELY"), dtype="string" + ), + "main_residence_value": value, + "property_wealth": value, + HIGH_VALUE_SUPPORT_COPY_COLUMN: np.asarray(copy, dtype=bool)[order], + "source_household_id": np.asarray(sources, dtype="int64")[order], + } + ) + benunit = pd.DataFrame({"benunit_id": ids, "frs_benunit_capital": 0.0}) + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "person_benunit_id": ids, + "age": np.full(len(ids), 50, dtype="int64"), + } + ) + return uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2024", + weight_kind=WeightKind.DESIGN, + household_weights=np.full(len(ids), 1_000.0), + ) + + +def test_the_cap_count_keeps_each_copy_in_its_clone(monkeypatch) -> None: + """A later copy keeps its clone's ordinal in the ids' geographic_support + field, so at one clone every row counts, copies included, as before, and + at two clones a copy counts only where it copies a clone-0 row. Exact + counts on synthetic rows: the band is read through as the count.""" + + parsed, knots = rhv.load_region_knots() + monkeypatch.setattr( + rhv, + "load_region_knots", + lambda payload=None: (dataclasses.replace(parsed, cap=1.0), knots), + ) + monkeypatch.setattr(rhv, "support_band", lambda count: count) + support, high_value, clone = 10**11, 10**12, 10**9 + # One clone: sources 2 and 3 own, and 1 through its high-value copy; the + # SPI copy of 2 rents. + one = _owners_by_id( + [1, 2, 3, 1 + high_value, 2 + support], + [1, 2, 3, 1, 2], + owner=[False, True, True, True, False], + copy=[False, False, False, True, False], + ) + assert rhv.assign_regional_home_values(one).receipt["sources_at_cap"] == 3 + # Two clones: clone 0 owns at source 3 only, and source 1 owns through its + # clone-0 row's copy; clone 1 and its copies own throughout. + two = _owners_by_id( + [ + 1, + 2, + 3, + clone + 1, + clone + 2, + clone + 3, + 1 + high_value, + clone + 2 + high_value, + clone + 3 + support, + ], + [1, 2, 3, 1, 2, 3, 1, 2, 3], + owner=[False, False, True, True, True, True, True, True, True], + copy=[False] * 6 + [True, True, False], + ) + assert rhv.assign_regional_home_values(two).receipt["sources_at_cap"] == 2 + + +def _graph_input_columns() -> set[str]: + """The household columns the production node is given.""" + + from microcosm.build.uk_runtime.graph import uk_spine_graph + + node = uk_spine_graph(source_mode="split").node(rhv.REGIONAL_HOME_VALUES_STAGE_NAME) + return { + column + for slice_ in node.inputs + if slice_.entity == "household" + for column in slice_.columns + } + + def test_the_stage_refuses_a_frame_without_lineage() -> None: with pytest.raises(ValueError, match="source_household_id"): rhv.assign_regional_home_values(_frame(lineage=False)) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_salary_sacrifice.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_salary_sacrifice.py index 9c017b3995..0ca26111b0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_salary_sacrifice.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_salary_sacrifice.py @@ -15,6 +15,7 @@ from microcosm.build.uk_runtime.cgt_structure import HOUSEHOLD_IS_CGT_CLONE from microcosm.build.uk_runtime.cgt_support import HOUSEHOLD_IS_CGT_SUPPORT_COPY from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_FOLD_EVIDENCE from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.salary_sacrifice import ( BONUS_SACRIFICE_OUTPUT, @@ -1159,3 +1160,29 @@ def test_the_stage_draws_each_clones_spi_bonuses_from_that_clone( for row in takers: donor = donor_rows[np.argmin(np.abs(shares - sacrificed[row] / pay[row]))] assert clone[donor] == clone[row] + + +def test_the_forests_fold_receipt_is_absent_at_one_clone_and_rounded_at_three( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The stage records how its forest's geographic fold met the asked + respondents only on more than one clone, so a one-clone frame's evidence + keeps its bytes; the counts are true source households, rounded + (microcosm#931).""" + + _Fitted.value = 0.0 + monkeypatch.setattr(salary_sacrifice, "QRF", _FakeQRF) + anchor = _toy_bonus_anchor(obr_base_above_threshold=1e12) + frame = _bonus_clone_frame() + for one in (frame, clone_first_frame(frame, 1)): + evidence = impute_salary_sacrifice(one, bonus_anchor=anchor).evidence() + assert GEOGRAPHIC_FOLD_EVIDENCE not in evidence + three = impute_salary_sacrifice(clone_first_frame(frame, 3), bonus_anchor=anchor) + # The 48 paid FRS respondents read 50; their clones agree on the forest's + # inputs. + assert three.evidence()[GEOGRAPHIC_FOLD_EVIDENCE] == { + "clone_count": 3, + "records": 50, + "varied_records": 0, + "varied_share": 0, + } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index 63509dfda8..fa19b8d158 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util +import json +import re from types import SimpleNamespace import numpy as np @@ -463,17 +465,25 @@ def test_stack_reserves_band_rows_and_conserves_every_region_mass() -> None: assert evidence["mass"]["old_total"] == evidence["mass"]["new_total"] # Every incumbent of a region is scaled by that region's one factor, and - # the zero-weight support copies stay at zero. + # the zero-weight support copies stay at zero. The result keeps each + # stratum's factor; the receipt gives the funding in two booleans and + # withholds every funding row's masses and factor (microcosm#931). incumbents = len(frame.table("household")) ratio = pd.Series(weights.to_numpy()[:incumbents] / np.asarray(incoming.values)) regions = pd.Series(frame.table("household")["region"].to_numpy()) positive = np.asarray(incoming.values) > 0.0 - factors = {row["stratum"]: row["factor"] for row in evidence["funding"]} + factors = {row["stratum"]: row["factor"] for row in result.funding_rows} for region, factor in factors.items(): assert 0.5 <= factor < 1.0 rows = (regions == region).to_numpy() & positive assert ratio[rows].to_numpy() == pytest.approx(factor, rel=1e-9) assert (weights.to_numpy()[:incumbents][~positive] == 0.0).all() + assert evidence["funding_within_share"] is True + assert evidence["funding_matches_bands"] is True + for row in evidence["funding"]: + assert row["stratum"] in factors + for key in ("incumbent_mass", "donor_mass", "factor"): + assert row[key] == UK_WITHHELD def test_spi_copies_and_donors_take_their_base_roots_ids_plus_the_radix() -> None: @@ -524,7 +534,9 @@ def test_stack_scales_seats_then_mass_on_a_frame_too_small_for_the_plan() -> Non ) evidence = result.evidence() assert evidence["expected_donor_count"] == 1_920 - assert evidence["donor_count"] == 9 + # The capped receipt counts the donors in rounded source households. + assert result.donor_count == result.donor_source_households == 9 + assert evidence["donor_count"] == rounded_count(9) assert result.seating_scale == pytest.approx(0.15 * 60 / 1_920) # The receipt says only that the cap bound: the exact scale gives back # the eligible household count. @@ -542,7 +554,8 @@ def test_stack_scales_seats_then_mass_on_a_frame_too_small_for_the_plan() -> Non ) evidence = heavy.evidence() assert 0.0 < evidence["mass_scale"] < 1.0 - assert min(row["factor"] for row in evidence["funding"]) == pytest.approx(0.5) + assert min(row["factor"] for row in heavy.funding_rows) == pytest.approx(0.5) + assert evidence["funding_within_share"] is True for row in evidence["bands"]: assert row["donor_weight"] == pytest.approx( row["planned_donor_weight"] * evidence["mass_scale"] @@ -1061,7 +1074,8 @@ def test_stage_transform_runs_on_a_synthetic_tape_and_receipts() -> None: # The survey-side sample fraction is provenance only: the seats follow # the frame, not the fraction. assert evidence["sample_fraction"] == 0.01 - assert evidence["donor_count"] == 9 and evidence["seating_scale"] == "capped" + assert evidence["seating_scale"] == "capped" + assert evidence["donor_count"] == rounded_count(9) assert [row["lower_bound"] for row in evidence["bands"]] == list(BANDS) # The carriers' regions, each count rounded on distinct source households. household = result.table("household") @@ -1528,7 +1542,7 @@ def test_one_mass_scale_serves_every_clone_when_one_clone_region_binds() -> None # then the binding stratum gives up exactly the declared share, and its # scale serves every band in every clone, so each band keeps one weight. - factors = {row["stratum"]: row["factor"] for row in evidence["funding"]} + factors = {row["stratum"]: row["factor"] for row in result.funding_rows} binding = min(factors, key=factors.get) assert 0.0 < evidence["mass_scale"] < 1.0 assert binding.endswith("|1") @@ -1552,25 +1566,31 @@ def test_one_mass_scale_serves_every_clone_when_one_clone_region_binds() -> None assert _mass_by_clone_and_region(result.frame) == pytest.approx( _mass_by_clone_and_region(light), rel=1e-12 ) - # The stage gate reconciles the scaled receipt band by band. + # The stage gate reconciles the scaled receipt band by band, and reads the + # binding stratum's funding, at the declared share to the gate's relative + # tolerance, from the receipt's booleans; its details give no factor. + assert evidence["funding_within_share"] is True + assert evidence["funding_matches_bands"] is True gate = _small_gate(evidence) assert gate.passed, gate.failures - assert gate.details["smallest_funding_factor"] == pytest.approx(0.5, rel=1e-12) + assert "smallest_funding_factor" not in gate.details + assert gate.details["funding_within_share"] is True -def test_a_clone_without_candidates_is_refused_by_count() -> None: +def test_a_clone_without_candidates_is_refused_without_a_count() -> None: frame = _clone_first_support_frame(200, 2) person = frame.table("person").copy() clone_of = frame.table("household").set_index("household_id")[_CLONE] in_clone_one = person["person_household_id"].map(clone_of).eq(1).to_numpy() person.loc[in_clone_one, "is_uc_claimant"] = False + # The refusal names the band and the clone's shortfall, never a count. with pytest.raises(ValueError) as caught: _stack_small(_with_tables(frame, person=person)) assert str(caught.value) == ( - "SPI income band donors need 3 candidate households with positive weight " - "and propensity for the band from 2000000 in each of the 2 geographic " - "clones; 1 clone(s) found fewer, the fewest 0." + "SPI income band donors found fewer candidate households with positive " + "weight and propensity for the band from 2000000 than the band seats in a " + "geographic clone." ) @@ -1690,6 +1710,8 @@ def test_the_receipt_counts_source_households_at_the_disclosure_floor() -> None: "expected_donor_count", "donor_count", "reallocated_mass", + "funding_within_share", + "funding_matches_bands", "mass", "draw", "taxpayer_count_resource", @@ -1713,6 +1735,8 @@ def test_the_receipt_counts_source_households_at_the_disclosure_floor() -> None: "carrier_mean_age", "carrier_regions", } + # At every seating a funding row keeps its stratum and rounded donors and + # withholds its masses and factor (microcosm#931). assert {key for row in evidence["funding"] for key in row} == { "stratum", "donor_households", @@ -1720,19 +1744,61 @@ def test_the_receipt_counts_source_households_at_the_disclosure_floor() -> None: "donor_mass", "factor", } + assert all( + row[key] == UK_WITHHELD + for row in evidence["funding"] + for key in ("incumbent_mass", "donor_mass", "factor") + ) # A band whose carriers come from fewer than three households withholds # their mean age: the published plan on 60 households seats one in three - # of the bands. - capped = stack_spi_income_band_donors( + # of the bands. The result's exact rows keep that rule. + capped_result = stack_spi_income_band_donors( _support_frame(), propensity=propensity, **_PUBLISHED - ).evidence() - for row in capped["bands"]: + ) + for row in capped_result.band_rows: if row["carriers"] < 3: assert row["carrier_mean_age"] == UK_WITHHELD else: assert isinstance(row["carrier_mean_age"], float) - assert sum(row["carriers"] < 3 for row in capped["bands"]) == 3 + assert sum(row["carriers"] < 3 for row in capped_result.band_rows) == 3 + + # The capped receipt gives each band's donors and carriers in rounded + # source households, withholds its weighted taxpayers and mean age, and + # leaves its carrier regions out; its donor count is rounded too, and its + # reallocated mass withheld (microcosm#931). + capped = capped_result.evidence() + assert capped["seating_scale"] == "capped" + assert capped["donor_count"] == rounded_count(capped_result.donor_source_households) + assert capped["reallocated_mass"] == UK_WITHHELD + assert capped["funding_within_share"] is capped["funding_matches_bands"] is True + for row, exact in zip(capped["bands"], capped_result.band_rows, strict=True): + assert set(row) == { + "lower_bound", + "expected_donor_households", + "donor_households", + "carriers", + "eligible_households", + "planned_donor_weight", + "donor_weight", + "weighted_taxpayers", + "published_taxpayers", + "carrier_mean_age", + } + assert row["donor_households"] == rounded_count( + exact["donor_source_households"] + ) + assert row["carriers"] == rounded_count(exact["carrier_source_households"]) + assert row["weighted_taxpayers"] == row["carrier_mean_age"] == UK_WITHHELD + for key in ( + "lower_bound", + "expected_donor_households", + "eligible_households", + "planned_donor_weight", + "donor_weight", + "published_taxpayers", + ): + assert row[key] == exact[key] def _one_top_band_candidate(person, household) -> pd.DataFrame: @@ -1770,19 +1836,20 @@ def _one_top_band_candidate(person, household) -> pd.DataFrame: ) -def test_a_band_short_of_candidates_on_one_clone_is_refused_by_count() -> None: +def test_a_band_short_of_candidates_on_one_clone_is_refused_without_a_count() -> None: # Given one claimant alone the tape lets carry the top band, on a spine # without the clone index, and three seats for the band, person, benunit, household = _raw_support_tables(200) propensity = _one_top_band_candidate(person, household) frame = _spi_support_frame(person, benunit, household, expected_spi_households=10) - # then the stage refuses with one clone's message: the seats and the count. + # then the stage refuses with one clone's message, which names the band + # and gives no count. with pytest.raises(ValueError) as caught: _stack_small(frame, propensity=propensity) assert str(caught.value) == ( - "SPI income band donors need 3 candidate households with positive weight " - "and propensity for the band from 2000000; found 1." + "SPI income band donors found fewer candidate households with positive " + "weight and propensity for the band from 2000000 than the band seats." ) @@ -1934,9 +2001,9 @@ def test_the_stage_gate_reads_a_receipt_by_its_clone_count() -> None: ) # Misread, three clones' donors split unevenly or outweigh the maximum. failures = " ".join(_small_gate({**evidence, "clone_count": 2}).failures) - assert "do not split evenly across the 2 geographic clones" in failures + assert "do not split evenly across the geographic clones" in failures failures = " ".join(_small_gate({**evidence, "clone_count": 4}).failures) - assert "divided by the geographic clone count 4" in failures + assert "exceeds the maximum divided by the geographic clone count" in failures # The gate spells the receipt's two seating forms itself. from microcosm.build.uk_runtime import stage_health @@ -1946,6 +2013,45 @@ def test_the_stage_gate_reads_a_receipt_by_its_clone_count() -> None: ) +def test_the_gate_spells_the_stages_columns_and_tolerances_as_the_stage_does() -> None: + # The stage-health gate imports no stage module, so it spells the band + # donor stage's columns, its copy-id field and the fields minted after it + # itself; and the stage's funding booleans compare at the gate's own + # tolerances (microcosm#931). + from microcosm.build.uk_runtime import spi_support, stage_health + + assert ( + stage_health._BAND_DONOR_FLAG_COLUMN, + stage_health._BAND_DONOR_BAND_COLUMN, + stage_health._BAND_CARRIER_FLAG_COLUMN, + stage_health._BAND_DONOR_CLONE_INDEX_COLUMN, + stage_health._BAND_DONOR_COPY_ID_FIELD, + ) == ( + HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, + SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN, + PERSON_IS_SPI_INCOME_BAND_CARRIER, + spi_support.SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN, + spi_support.SPI_SUPPORT_COPY_ID_FIELD, + ) + assert stage_health._COPY_ID_FIELDS_AFTER_BAND_DONORS == _LATER_FIELDS + assert ( + spi_band_donors.SPI_INCOME_BAND_FUNDING_RELATIVE_TOLERANCE + == stage_health._FLOAT_RELATIVE_TOLERANCE + ) + (entry,) = [ + entry + for entry in load_country_spec("uk").gates.gates + if entry.gate == "stage_health" + and entry.parameters["check"] == "spi_income_band_donor_support" + ] + assert spi_band_donors.SPI_INCOME_BAND_FUNDING_MASS_TOLERANCE == max( + entry.parameters["maximum_relative_mass_deviation"], + stage_health._FLOAT_RELATIVE_TOLERANCE, + ) + share = spi_band_donors.SPI_INCOME_BAND_MAXIMUM_REALLOCATION_SHARE + assert 1.0 - share >= entry.parameters["minimum_funding_factor"] + + def test_e8_receipt_replays_the_seating_of_every_clone() -> None: tool = _identity_tool() propensity = _propensity() @@ -2326,3 +2432,209 @@ def test_the_e8_entry_hands_the_band_donor_check_the_artifact(monkeypatch) -> No assert receipt["stored_column_mismatches"] == {} assert receipt["band_donors"]["seating_recomputed"] is True assert receipt["band_donors"]["donor_households"] == stage_17.donor_count + + +# The stage gate on a capped seating (microcosm#931). A capped receipt gives +# each band's donors and carriers only in rounded source households, so the +# gate recomputes the seats from the transferred phase's frame: the band +# donor rows that are originals in every copy-id field minted after the band +# donors. Every frame below is synthetic. + +#: The stage gate's declaration at the published plan's constants. +_PUBLISHED_GATE_PARAMETERS = { + **_SMALL_GATE_PARAMETERS, + "minimum_donors_per_band": SPI_INCOME_BAND_MINIMUM_DONORS, + "maximum_donor_weight": SPI_INCOME_BAND_MAXIMUM_DONOR_WEIGHT, +} + + +def _capped_gate(evidence, frame): + return uk_stage_health_gate( + evidence=evidence, + stage="spi_income_band_donors", + check="spi_income_band_donor_support", + parameters=_PUBLISHED_GATE_PARAMETERS, + frame=frame, + ) + + +def _no_data_digit(message: str) -> bool: + """Whether ``message`` holds no digit beyond the declared band bounds.""" + + bounds = "|".join(str(band) for band in sorted(BANDS, reverse=True)) + return re.search(r"\d", re.sub(rf"\b(?:{bounds})\b", "", message)) is None + + +def _capped_pair_frame(*, last_adult_claims: bool): + """81 households, whose last adult is a claimant or not. + + The published plan's capped seating cuts its seats from the eligible + households, so the frame whose last adult claims seats one more donor + in its first band: 81 and 80 eligible households sit either side of a + whole seat. + """ + + person, benunit, household = _raw_support_tables(81) + if not last_adult_claims: + last = person["person_household_id"].eq(81) & person["is_uc_claimant"] + person.loc[last, "is_uc_claimant"] = False + return _spi_support_frame(person, benunit, household, expected_spi_households=10) + + +def test_two_capped_frames_whose_seats_differ_by_one_give_one_receipt_and_verdict() -> ( + None +): + # Given two frames of the same households and weights, one with one more + # eligible claimant, whose capped seatings differ by one donor at a mass + # scale of one, + propensity = _propensity() + fewer, more = ( + stack_spi_income_band_donors( + _capped_pair_frame(last_adult_claims=claims), + propensity=propensity, + sample_fraction=0.01, + **_PUBLISHED, + ) + for claims in (False, True) + ) + assert more.donor_count == fewer.donor_count + 1 + assert fewer.seating_scale < 1.0 and more.seating_scale < 1.0 + assert fewer.mass_scale == more.mass_scale == 1.0 + + # then their receipts are byte-identical, and so are the gate's verdicts + # on the frames each stage wrote, whose details carry no count. + receipts = [ + json.dumps(result.evidence(), sort_keys=True) for result in (fewer, more) + ] + assert receipts[0] == receipts[1] + # So are the results' reprs, which leave out the exact figures and the + # frame. + assert repr(fewer) == repr(more) + for name in ( + "frame", + "band_rows", + "funding_rows", + "seating_scale", + "donor_count", + "donor_source_households", + "reallocated_mass", + ): + assert f"{name}=" not in repr(fewer).replace("expected_donor_count=", ""), name + gates = [_capped_gate(result.evidence(), result.frame) for result in (fewer, more)] + assert gates[0].passed, gates[0].failures + assert gates[0] == gates[1] + serialized = [ + json.dumps( + [gate.name, gate.passed, list(gate.failures), dict(gate.details)], + sort_keys=True, + ) + for gate in gates + ] + assert serialized[0] == serialized[1] + assert set(gates[0].details) == { + "stage", + "check", + "bands_checked", + "clone_count", + "seating_scale", + "mass_scale", + "funding_within_share", + "funding_matches_bands", + } + + +def test_the_capped_gate_reads_each_seat_once_through_every_later_copy() -> None: + # Given stage 17's capped seating on three clones, carried through every + # layer stacked after it, which copies donors in copy-id fields above the + # band donors' own, + propensity = _propensity() + stage_17, built = _stage_17_and_artifact(3, propensity) + evidence = {**stage_17.evidence(), "sample_fraction": 0.01} + assert evidence["seating_scale"] == "capped" + household = built.artifact.table("household") + donors = household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR].to_numpy() + copied_later = _stage_17_ids(household) != household["household_id"].to_numpy() + assert (donors & copied_later).any() + + # whose donor rows, counted with their later copies, split unevenly + # across the clones, + naive = ( + household.loc[donors] + .groupby([SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN, _CLONE]) + .size() + .unstack(fill_value=0) + ) + assert (naive.nunique(axis=1) > 1).any() + + # then the gate reads each band's seats as the stage wrote them, evenly + # split and each holding its one carrier, on the artifact as on the + # frame stage 17 wrote. + for frame in (stage_17.frame, built.artifact): + gate = _capped_gate(evidence, frame) + assert gate.passed, gate.failures + assert gate.details["clone_count"] == 3 + assert gate.details["seating_scale"] == "capped" + + +def test_the_capped_gate_fails_a_dropped_carrier_and_an_uneven_split() -> None: + propensity = _propensity() + stage_17, _ = _stage_17_and_artifact(3, propensity) + evidence = {**stage_17.evidence(), "sample_fraction": 0.01} + frame = stage_17.frame + household = frame.table("household") + person = frame.table("person") + band_of = household.set_index("household_id")[ + SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN + ] + messages: list[str] = [] + + # A seat whose carrier lost its flag holds no carrier. + carriers = np.flatnonzero(person[PERSON_IS_SPI_INCOME_BAND_CARRIER].to_numpy()) + dropped = person.copy() + dropped.loc[carriers[0], PERSON_IS_SPI_INCOME_BAND_CARRIER] = False + band = int(band_of[person.loc[carriers[0], "person_household_id"]]) + gate = _capped_gate(evidence, _with_tables(frame, person=dropped)) + assert not gate.passed + assert gate.failures == ( + f"spi_income_band_donors: band from {band}'s seats do not each hold one " + "carrier.", + ) + messages.extend(gate.failures) + + # A seat of clone 1 that is no longer a donor leaves its band uneven + # across the clones. + seats = household.loc[household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR]] + seat = seats.loc[seats[_CLONE].eq(1)].iloc[0] + unseated = household.copy() + unseated.loc[ + unseated["household_id"].eq(seat["household_id"]), + HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, + ] = False + band = int(seat[SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN]) + gate = _capped_gate(evidence, _with_tables(frame, household=unseated)) + assert not gate.passed + assert gate.failures == ( + f"spi_income_band_donors: band from {band}'s donors do not split evenly " + "across the geographic clones.", + ) + messages.extend(gate.failures) + + # A receipt read against another clone count splits unevenly too, and a + # capped receipt without the frame is not passed. + gate = _capped_gate({**evidence, "clone_count": 2}, frame) + assert not gate.passed + messages.extend(gate.failures) + gate = _capped_gate(evidence, None) + assert gate.failures == ( + "spi_income_band_donors: a capped seating's seats are recomputed from the " + "phase's frame, and the gate was given none.", + ) + messages.extend(gate.failures) + + # No message gives a count, a weight or a mass, and no capped verdict's + # details give the donors, their mass or their heaviest weight. + assert messages and all(_no_data_digit(message) for message in messages) + for verdict in (gate, _capped_gate(evidence, frame)): + assert not {"donor_count", "reallocated_mass", "heaviest_donor_weight"} & set( + verdict.details + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_employment_shell.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_employment_shell.py index 543759a92f..9c53cf684c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_employment_shell.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_employment_shell.py @@ -10,6 +10,7 @@ import pytest from microcosm.build import load_country_spec +from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_FOLD_EVIDENCE from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.spi_employment_shell import ( UK_SPI_EMPLOYMENT_SHELL_OPERATION, @@ -353,3 +354,39 @@ def test_geographic_clones_fold_into_each_survey_record(tables, shell) -> None: cloned_receipt["category_shares"]["training"] == receipt["category_shares"]["training"] ) + + +def test_the_shells_fold_receipt_is_absent_at_one_clone_and_rounded_at_three( + tables, shell +) -> None: + """The stage records how its fit's geographic fold met the FRS adults + only on more than one clone, so a one-clone frame's receipt keeps its + bytes; the counts are true source households, rounded (microcosm#931).""" + + person, household, weights = tables + _, receipt = shell + assert GEOGRAPHIC_FOLD_EVIDENCE not in receipt + pool = with_geographic_clones( + uk_national_frame( + person=person.assign(person_benunit_id=person["person_household_id"]), + benunit=pd.DataFrame({"benunit_id": household["household_id"].to_numpy()}), + household=household, + household_weights=weights, + time_period="2024", + ), + n_clones=3, + ) + _, cloned = impute_spi_employment_shell( + pool.table("person"), + pool.table("household"), + pool.weights_for("household").values, + boosted_max_iter=60, + ) + # The 1,500 FRS adults train the shell, one per household; their clones + # agree. + assert cloned[GEOGRAPHIC_FOLD_EVIDENCE] == { + "clone_count": 3, + "records": 1500, + "varied_records": 0, + "varied_share": 0, + } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_housing_shell.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_housing_shell.py index dd56734a98..2c1b938272 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_housing_shell.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_housing_shell.py @@ -12,6 +12,7 @@ UK_COUNCIL_TAX_BANDS, UK_COUNCIL_TAX_BANDS_BY_REGION, ) +from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_FOLD_EVIDENCE from microcosm.build.uk_runtime.national_frame import uk_national_frame from microcosm.build.uk_runtime.spi_housing_shell import ( HOUSING_AMOUNT_COLUMNS, @@ -332,3 +333,40 @@ def test_geographic_clones_train_the_model_as_their_survey_household() -> None: column: shares["training"] for column, shares in alone_receipt["category_shares"].items() } + + +def test_the_models_fold_receipt_is_absent_at_one_clone_and_rounded_at_three() -> None: + """The stage records how its model's geographic fold met the FRS + households only on more than one clone, so a one-clone frame's receipt + keeps its bytes; the counts are true source households, rounded + (microcosm#931).""" + + person, benunit, household, weights = housing_shell_tables() + _, _, alone = impute_spi_housing_shell( + person, household, weights, seed=0, n_estimators=5 + ) + assert GEOGRAPHIC_FOLD_EVIDENCE not in alone + pool = with_geographic_clones( + uk_national_frame( + person=person, + benunit=benunit, + household=household, + household_weights=weights, + time_period="2024", + ), + n_clones=3, + ) + _, _, receipt = impute_spi_housing_shell( + pool.table("person"), + pool.table("household"), + pool.weights_for("household").values, + seed=0, + n_estimators=5, + ) + # The 600 FRS households train the model; their clones agree. + assert receipt[GEOGRAPHIC_FOLD_EVIDENCE] == { + "clone_count": 3, + "records": 600, + "varied_records": 0, + "varied_share": 0, + } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_income.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_income.py index fc49822fb9..03c51484cb 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_income.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_income.py @@ -3,7 +3,15 @@ # ruff: noqa: F403, F405 from dataclasses import replace as _replace +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.national_frame import uk_national_frame as _uk_frame +from microcosm.build.uk_runtime.spi_rank_transport import ( + cell_fold_ids, + cross_fit_transport, + identity_uniforms, +) +from microcosm.build.uk_runtime.spi_support import support_source_id_column +from test_support.microcosm_build.uk_clone_first_frames import GEOGRAPHIC_RADIX from test_support.microcosm_build.uk_geographic_clones import with_geographic_clones from test_support.microcosm_build.uk_spi_income import * @@ -1535,6 +1543,50 @@ def _geographic_clones_of(support): ) +#: The id columns a geographic clone mints, with each copy's source id: the +#: support channel runs after the geographic expansion on the clone-first +#: spine, so each clone's copies copy that clone's own FRS rows. +_CLONE_MINTED_COLUMNS = { + "person": ("person_id", "person_household_id", "person_benunit_id"), + "benunit": ("benunit_id",), + "household": ("household_id",), +} + + +def _clone_first_support(support, n_clones: int): + """The support as the clone-first spine makes it, at ``n_clones`` clones. + + Clone ``k`` repeats every row, SPI copies included, with its ids and its + copy's source id minted at ``k`` in the copy-id table's + ``geographic_support`` field, its household's ``household_clone_index`` + ``k``, the same ``source_household_id`` and its household's weight over + ``n_clones``. Each table is in id order, as the spine stacks the support + channel's copies after every clone's FRS rows. + """ + + tables = {} + for entity, columns in _CLONE_MINTED_COLUMNS.items(): + table = getattr(support, entity) + clones = [] + for k in range(n_clones): + clone = table.copy() + for column in (*columns, support_source_id_column(entity)): + if k: + clone[column] = copy_id_table.mint( + clone[column], "geographic_support", k + ) + if entity == "household": + clone["household_clone_index"] = np.full(len(clone), k, np.int64) + clone["household_weight"] = clone["household_weight"] / n_clones + clones.append(clone) + tables[entity] = ( + pd.concat(clones, ignore_index=True) + .sort_values(columns[0], kind="stable") + .reset_index(drop=True) + ) + return _replace(support, **tables) + + def test_a_household_table_carrying_the_artifact_clone_index_is_refused( monkeypatch, tmp_path ) -> None: @@ -1612,3 +1664,141 @@ def frs_fits(frame): assert (cloned_kind, cloned_gate) == (kind, gate) np.testing.assert_array_equal(cloned_groups, groups) np.testing.assert_array_equal(cloned_weights, weights) + + +def _impute_recording_transport(monkeypatch, frame, donor_path, *, translate=True): + """Run the stage on ``frame``, recording what it hands the transport. + + Returns the stage's result, the transport call's keyword arguments and + the transport's result. ``translate=False`` drops the twin identities + the stage passes, so the twin check reads each copy's own identity, as + it did before the stage translated it through its fold. + """ + + calls = [] + + def recording(*args, **kwargs): + kwargs = {**kwargs, "allow_in_sample": True} + if not translate: + kwargs.pop("twin_identities") + transport = cross_fit_transport(*args, **kwargs) + calls.append((kwargs, transport)) + return transport + + monkeypatch.setattr(spi_income, "cross_fit_transport", recording) + result = impute_uk_spi_income_support( + frame, + donor_path, + seed=9, + n_estimators=3, + donor_sample_size=None, + state_pension_age=66, + allow_in_sample_reference=True, + stage1_gate_calibration="cross_fitted", + ) + ((kwargs, transport),) = calls + return result, kwargs, transport + + +def _same_transport(first, second) -> bool: + return first.receipt == second.receipt and all( + first.quantiles[target].tobytes() == second.quantiles[target].tobytes() + and first.sign_uniforms[target].tobytes() + == second.sign_uniforms[target].tobytes() + for target in SPI_INCOME_QRF_OUTPUT_COLUMNS + ) + + +def test_the_forests_fold_receipt_is_absent_at_one_clone_and_rounded_at_three( + monkeypatch, tmp_path +) -> None: + """The stage records how its forests' geographic fold met their survey + records only on more than one clone, counted in true source households + and rounded (microcosm#931).""" + + support = _dead_support() + donor_path = tmp_path / SPI_DONOR_FILENAME + _write_donor(donor_path) + monkeypatch.setattr(spi_income, "QRF", _FakeQRF) + _bypass_reviewed_donor_identity(monkeypatch) + alone, _, _ = _impute_recording_transport(monkeypatch, support, donor_path) + assert alone.geographic_fold is None + one, _, _ = _impute_recording_transport( + monkeypatch, _clone_first_support(support, 1), donor_path + ) + assert one.geographic_fold is None + three, _, _ = _impute_recording_transport( + monkeypatch, _clone_first_support(support, 3), donor_path + ) + # Four FRS households train the forests; no clone differs from another. + assert three.geographic_fold == { + "clone_count": 3, + "records": "3-9", + "varied_records": 0, + "varied_share": 0, + } + + +def test_the_twin_check_reaches_every_clones_copies(monkeypatch, tmp_path) -> None: + """On the clone-first spine each clone's SPI copies copy that clone's own + FRS rows, and the forests train on clone 0's rows, which the other + clones fold into. The stage translates each copy's twin through the fold + for the twin check alone: clone 1's copies are checked against their + twin's fold too, and a fold-key cell moved on one of them is refused, + while every copy's uniforms stay keyed on its own twin, so clone 1 draws + apart from clone 0. On one clone the two keys are the same and the + transport is the one the untranslated check gives (microcosm#931).""" + + support = _dead_support() + donor_path = tmp_path / SPI_DONOR_FILENAME + _write_donor(donor_path) + monkeypatch.setattr(spi_income, "QRF", _FakeQRF) + _bypass_reviewed_donor_identity(monkeypatch) + + alone, kwargs, transport = _impute_recording_transport( + monkeypatch, support, donor_path + ) + assert kwargs["twin_identities"].tobytes() == kwargs["identities"].tobytes() + untranslated, _, before = _impute_recording_transport( + monkeypatch, support, donor_path, translate=False + ) + assert _same_transport(transport, before) + pd.testing.assert_frame_equal(alone.person, untranslated.person, check_exact=True) + + two = _clone_first_support(support, 2) + _, kwargs, transport = _impute_recording_transport(monkeypatch, two, donor_path) + identities, twins = kwargs["identities"], kwargs["twin_identities"] + clone_one = identities >= GEOGRAPHIC_RADIX + assert int(clone_one.sum()) == int((~clone_one).sum()) == 4 + # Each copy keeps its own twin's id; the check reads clone 1's twins on + # the clone-0 rows they folded into, which the forests train on. + assert twins[~clone_one].tolist() == identities[~clone_one].tolist() + assert ( + twins[clone_one].tolist() == (identities[clone_one] - GEOGRAPHIC_RADIX).tolist() + ) + assert set(twins.tolist()) <= set(kwargs["training_identities"].tolist()) + for target in SPI_INCOME_QRF_OUTPUT_COLUMNS: + placed = identity_uniforms(identities, target=target, part="quantile", seed=9) + np.testing.assert_array_equal(transport.quantiles[target], placed) + assert not np.isin(placed[clone_one], placed[~clone_one]).any() + + # A gender flip that moves a clone-1 copy off its twin's fold. + predictors = kwargs["recipient_predictors"] + fold_call = { + "folds": kwargs["folds"], + "seed": kwargs["seed"], + "used": kwargs["used"], + } + flipped = predictors["gender"].map({"MALE": "FEMALE", "FEMALE": "MALE"}) + folds, _ = cell_fold_ids(predictors, **fold_call) + moved, _ = cell_fold_ids(predictors.assign(gender=flipped), **fold_call) + candidates = np.flatnonzero(clone_one & (moved != folds)) + assert candidates.size + label = predictors.index[candidates[0]] + person = two.person.copy() + person.loc[label, "gender"] = flipped.loc[label] + mutated = _replace(two, person=person) + # Before the translation the check could not see clone 1's twins. + _impute_recording_transport(monkeypatch, mutated, donor_path, translate=False) + with pytest.raises(ValueError, match="differs from its twin's"): + _impute_recording_transport(monkeypatch, mutated, donor_path) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_rank_transport.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_rank_transport.py index db8f7878ae..bb32b5aacf 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_rank_transport.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_rank_transport.py @@ -6,6 +6,7 @@ import pandas as pd import pytest +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime import spi_rank_transport as transport from microcosm.build.uk_runtime.receipt_counts import rounded_count from microcosm.build.uk_runtime.spi_rank_transport import ( @@ -13,6 +14,8 @@ analogue_source_columns, analogue_targets, assert_analogue_map_covers, + cell_fold_ids, + cross_fit_transport, identity_uniforms, observed_analogues, transport_uniforms, @@ -551,3 +554,128 @@ def fit_reference(encoded_train, observed_train): seed=42, folds=5, ) + + +def _two_clone_call() -> tuple[dict[str, object], np.ndarray]: + """A cross-fit call over the copies of two geographic clones. + + The training rows are clone 0's FRS rows, since the stage folds each + record's clone 1 onto its clone 0 where they agree. The first forty + rows have a copy in each clone; each copy's identity is its own twin's + id, clone 1's minted in the copy-id table's ``geographic_support`` + field, so clone 1's twins are not among the training rows. Returns the + call and each copy's twin identity: the training row that holds its twin. + """ + + rng = np.random.default_rng(13) + n = 300 + regions = np.array(["LONDON", "WALES", "SCOTLAND", "NORTH_EAST", "SOUTH_WEST"]) + raw = pd.DataFrame( + { + "gender": np.where(rng.random(n) < 0.5, "MALE", "FEMALE"), + "region": regions[rng.integers(0, len(regions), n)], + "spi_age_band": rng.integers(2, 6, n).astype(float), + "main_source": "pay", + "has_pay": 1.0, + "has_trade": 0.0, + "at_or_over_spa": 0.0, + } + ) + encoded = pd.DataFrame({"spi_age_band": raw["spi_age_band"]}) + observed = pd.DataFrame(index=encoded.index) + for target in analogue_targets(): + observed[target] = 1_000.0 + training_folds, used = cell_fold_ids(raw, folds=5, seed=42) + assert used == 5 + twins = np.arange(10_000, 10_000 + n, dtype=np.int64) + copied = np.r_[np.arange(40), np.arange(40)] + call = { + "training_predictors": raw, + "encoded_training": encoded, + "training_observed": observed, + "training_folds": training_folds, + "training_identities": twins, + "used": used, + "recipient_predictors": raw.iloc[copied].reset_index(drop=True), + "encoded_recipients": encoded.iloc[copied].reset_index(drop=True), + "observed": observed.iloc[copied].reset_index(drop=True), + "identities": np.r_[ + twins[:40], copy_id_table.mint(twins[:40], "geographic_support", 1) + ], + "groups": None, + "seed": 42, + "folds": 5, + } + return call, twins[copied] + + +def _placing_reference(): + """A reference forest that locates nothing: it hands the placements back.""" + + reference = _ClassReference([(0, 1)], analogue_targets()) + + def fit_reference(encoded_train, observed_train): + reference._fits += 1 + return reference + + return fit_reference + + +def _same_result(first, second) -> bool: + return ( + all( + first.quantiles[target].tobytes() == second.quantiles[target].tobytes() + and first.sign_uniforms[target].tobytes() + == second.sign_uniforms[target].tobytes() + for target in SPI_INCOME_QRF_OUTPUT_COLUMNS + ) + and first.receipt == second.receipt + ) + + +def test_twin_identities_reach_every_clones_copies_and_key_no_uniform() -> None: + """The twin check finds each copy's twin by its twin identity, which the + SPI income stage translates through its geographic fold, so the copies + of clone 1, whose twins folded into clone 0's rows, are checked too. + Every copy's uniforms stay keyed on its own identity, so clone 1's copies + draw apart from clone 0's (microcosm#931).""" + + call, twin_identities = _two_clone_call() + identities = call["identities"] + fit_reference = _placing_reference() + default = cross_fit_transport(fit_reference, **call) + # Without folded twins the two arrays are equal, and nothing changes. + same = cross_fit_transport(fit_reference, **call, twin_identities=identities) + assert _same_result(same, default) + translated = cross_fit_transport( + fit_reference, **call, twin_identities=twin_identities + ) + assert _same_result(translated, default) + for target in translated.receipt["identity_targets"]: + np.testing.assert_array_equal( + translated.quantiles[target], + identity_uniforms(identities, target=target, part="quantile", seed=42), + ) + assert ( + translated.quantiles[target][40:] != translated.quantiles[target][:40] + ).all() + # A fold-key cell of a clone-1 copy that moves the copy off its twin's + # fold: the check misses it without the translation, which leaves the + # copy's twin unknown, and refuses it with the translation. + folds, _ = cell_fold_ids( + call["recipient_predictors"], folds=5, seed=42, used=call["used"] + ) + moved = call["recipient_predictors"].copy() + row = 40 + for region in ("EAST_MIDLANDS", "WEST_MIDLANDS", "YORKSHIRE", "EAST_OF_ENGLAND"): + moved.loc[row, "region"] = region + shifted, _ = cell_fold_ids(moved, folds=5, seed=42, used=call["used"]) + if shifted[row] != folds[row]: + break + assert shifted[row] != folds[row] + mutated = {**call, "recipient_predictors": moved} + cross_fit_transport(fit_reference, **mutated) + with pytest.raises(ValueError, match="differs from its twin's"): + cross_fit_transport(fit_reference, **mutated, twin_identities=twin_identities) + with pytest.raises(ValueError, match="twin identities must have one entry"): + cross_fit_transport(fit_reference, **call, twin_identities=twin_identities[:-1]) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py index a9ea65056b..9bed95ac24 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py @@ -20,6 +20,7 @@ HMRCIncomeTargetSet, ) from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.receipt_counts import rounded_count from microcosm.build.uk_runtime.spi_income import ( FRS_ONLY_SPI_FILL_PERSON_COLUMNS, SPI_DONOR_REQUIRED_COLUMNS, @@ -1286,6 +1287,74 @@ def copied(household, clone: int) -> set[int]: ) +@pytest.mark.parametrize("clone_count", [3, 7]) +def test_stage_16_counts_its_pension_age_households_in_sources_at_any_clone_count( + clone_count: int, +) -> None: + """The receipt's two pension-age counts are of survey households, rounded + (microcosm#931): the pension-age base households' distinct sources, which + every clone shares, and clone 0's pension-age SPI copies' distinct + sources, which a one-clone spine copies alike. So both are the same at + every clone count where the share is below one, and the receipt gives the + clone count beside them; the masses are as they were.""" + + transforms = {} + evidence = {} + frames = {} + for clones in (1, clone_count): + transform = UKSPISupportChannelStageTransform( + stage=_committed_stage("spi_support_channel"), + sample_fraction=0.01, + state_pension_age=66, + ) + frames[clones] = transform( + expand_uk_geographic_clones(_share_frame(), clones=clones).frame + ) + transforms[clones] = transform + evidence[clones] = transform.checkpoint_metadata()["evidence"] + one = transforms[1].last_result.pension_age_allocation + many = transforms[clone_count].last_result.pension_age_allocation + counts = ("pension_age_households", "pension_age_spi_households") + + # 10,000 x 0.01 expected copies over 400 sources: a share of 1/4. + assert evidence[clone_count]["inclusion_probability"] == 0.25 + assert evidence[1]["clone_count"] == 1 + assert evidence[clone_count]["clone_count"] == clone_count + for key in counts: + assert many[key] == one[key] > 0 + assert evidence[clone_count]["pension_age_allocation"][key] == rounded_count( + one[key] + ) + assert evidence[1]["pension_age_allocation"][key] == rounded_count(one[key]) + for key in set(one) - set(counts): + assert many[key] == pytest.approx(one[key], rel=1e-12) + assert evidence[clone_count]["pension_age_allocation"][key] == many[key] + + # Counted in rows, the base households would grow with the clone count, + # and the SPI copies of every clone together count more sources than + # clone 0's: each clone draws its copies on its own ids. + household = frames[clone_count].table("household") + person = frames[clone_count].table("person") + pension_age = household["household_id"].isin( + person.loc[person["age"].ge(66), "person_household_id"] + ) + channel = household["household_support_channel"] + base = channel.ne(SPI_SYNTHETIC_SUPPORT_CHANNEL) + copies = channel.eq(SPI_SYNTHETIC_SUPPORT_CHANNEL) + assert int((pension_age & base).sum()) == clone_count * one[counts[0]] + assert ( + household.loc[pension_age & copies, "source_household_id"].nunique() + > one[counts[1]] + ) + assert ( + household.loc[ + pension_age & copies & household["household_clone_index"].eq(0), + "source_household_id", + ].nunique() + == one[counts[1]] + ) + + def test_the_support_channel_writes_no_source_column_of_its_own() -> None: """The households' source columns are geographic_expand's (microcosm#931). @@ -1328,3 +1397,103 @@ def test_the_support_channel_writes_no_source_column_of_its_own() -> None: household.loc[spi, source].reset_index(drop=True), roots.loc[household.loc[spi, "household_source_id"]].reset_index(drop=True), ) + + +def _income_stage_evidence( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, frame +) -> dict[str, object]: + """Stage 18's checkpoint evidence on ``frame``, after the support channel. + + The forests, the uprating and the engine reads are the engine-free + fixtures of test_spi_spine_parsed_inputs_match_the_path_resolution, and + the donor and the targets are handed to the stage parsed. + """ + + _FakeQRF.events = [] + monkeypatch.setattr(spi_income, "QRF", _FakeQRF) + monkeypatch.setattr( + spi_income, + "_refresh_disability_derived_inputs", + lambda person, *, spi_people, category_rates=None, flag_rates=None: person, + ) + monkeypatch.setattr( + spi_income, + "_spi_income_uprating_factors", + lambda year, *, rebased_columns=None: ( + {column: 1.25 for column in SPI_INCOME_QRF_OUTPUT_COLUMNS}, + {"from_period": 2022, "to_period": year, "basis": "synthetic_test"}, + ), + ) + monkeypatch.setattr( + spi_spine, + "load_spi_donor_age_model", + lambda _build_period: SPIDonorAgeModel( + populations={ + (sex, age): 1.0 for sex in ("MALE", "FEMALE") for age in range(0, 91) + }, + state_pension_age=66, + source="engine-free fixture", + ), + ) + monkeypatch.setattr( + spi_spine, + "load_spi_employment_pay_floor", + lambda _build_period: SPIEmploymentPayFloor( + age_thresholds=(0.0, 18.0, 21.0), + hourly_rates=(6.40, 8.60, 11.44), + weeks_in_year=52.0, + instant="2024-04-30", + source="engine-free fixture", + ), + ) + support_frame = UKSPISupportChannelStageTransform( + stage=_committed_stage("spi_support_channel"), + sample_fraction=0.0002, + state_pension_age=66, + )(frame) + donor_path = _donor_file(tmp_path) + ods_path = tmp_path / "synthetic-hmrc.ods" + ods_path.write_bytes(b"synthetic hmrc source") + transform = UKSPIIncomeSpineStageTransform( + donor_path, + ods_path, + stage=_with_mutated_operation( + _committed_stage("hmrc_spi_income_spine"), + "fit_weighted_qrf_stage1", + sample_size=1, + ), + donor_sample_size=1, + sampled_rung=True, + donor_table=pd.read_csv(donor_path, delimiter="\t"), + source_targets=_synthetic_hmrc_targets(ods_path), + ) + transform(support_frame) + return transform.checkpoint_metadata()["evidence"] + + +@pytest.mark.parametrize("clones", [None, 1, 3]) +def test_stage_18_records_its_forests_fold_only_on_more_than_one_clone( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, clones: int | None +) -> None: + """The income stage's evidence gives how its forests' geographic fold met + their survey records only on a frame of more than one clone, counted in + rounded distinct source households; one clone's evidence is as it was, + with or without the clone index (microcosm#931).""" + + from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_FOLD_EVIDENCE + + frame = _base_frame(time_period=HMRC_SPI_BUILD_PERIOD) + if clones is not None: + frame = expand_uk_geographic_clones(frame, clones=clones).frame + evidence = _income_stage_evidence(monkeypatch, tmp_path, frame) + + if clones in (None, 1): + assert GEOGRAPHIC_FOLD_EVIDENCE not in evidence + return + # Two FRS households train the forests, and no clone differs from another. + assert evidence[GEOGRAPHIC_FOLD_EVIDENCE] == { + "clone_count": 3, + "records": "<3", + "varied_records": 0, + "varied_share": 0, + } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py index 05c7773bda..582b6cb415 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_support.py @@ -920,6 +920,8 @@ def test_build_spi_support_channel_gives_pension_age_households_their_own_share( assert spi_support.SPI_PENSION_AGE_STRATUM_COLUMN not in result.household assert result.pension_age_spi_prior_mass_share == 0.2 assert result.state_pension_age == 66 + # The two counts are distinct source households: the pension-age base + # households', and clone 0's pension-age SPI copies' (microcosm#931). assert result.pension_age_allocation == { "pension_age_households": 2, "pension_age_spi_households": 2, @@ -928,8 +930,119 @@ def test_build_spi_support_channel_gives_pension_age_households_their_own_share( "other_prior_mass": 70.0, "other_spi_mass": pytest.approx(35.0), } + assert result.clone_count == 1 assert "0.2 in households with" in result.mass_log[-1].reason + # As two geographic clones, every household copied in each: the rows + # double, the source households and clone 0's copies do not, and the + # masses are the one clone's. + cloned = household.copy() + cloned["source_household_id"] = cloned["household_id"] + cloned["source_year"] = 2023 + cloned["source_household_key"] = [ + f"2023:{value}" for value in cloned["household_id"] + ] + cloned["household_clone_index"] = 0 + second = cloned.assign( + household_id=cloned["household_id"] + 10**9, + household_weight=cloned["household_weight"], + household_clone_index=1, + ) + two_clones = build_uk_spi_support_channel( + person=pd.concat( + [ + person, + person.assign( + person_id=person["person_id"] + 10**9, + person_household_id=person["person_household_id"] + 10**9, + person_benunit_id=person["person_benunit_id"] + 10**9, + ), + ], + ignore_index=True, + ), + benunit=pd.concat( + [benunit, benunit.assign(benunit_id=benunit["benunit_id"] + 10**9)], + ignore_index=True, + ), + household=pd.concat([cloned, second], ignore_index=True).assign( + household_weight=lambda table: table["household_weight"] / 2.0 + ), + expected_spi_households=4, + seed=42, + source_year=2023, + strata_columns=("region",), + pension_age_share=0.2, + state_pension_age=66, + ) + assert two_clones.clone_count == 2 + stacked = two_clones.household + copies = stacked[support_channel_column("household")].eq( + SPI_SYNTHETIC_SUPPORT_CHANNEL + ) + assert int(copies.sum()) == 8 + assert two_clones.pension_age_allocation == { + **result.pension_age_allocation, + "pension_age_spi_mass": pytest.approx(6.0), + "other_spi_mass": pytest.approx(35.0), + } + + +def test_the_pension_age_counts_take_the_carriers_of_their_masses() -> None: + """Each pension-age count is of the source households carrying the mass + printed beside it, as every receipt counts them: a base household of zero + prior weight carries none of the prior mass, while its SPI copy, which + the allocation weights, carries copy mass (microcosm#931).""" + + person, benunit, household = _pension_age_frames() + household = pd.concat( + [ + household, + pd.DataFrame( + {"household_id": [5], "household_weight": [0.0], "region": ["LONDON"]} + ), + ], + ignore_index=True, + ) + person = pd.concat( + [ + person, + pd.DataFrame( + { + "person_id": [5001], + "person_household_id": [5], + "person_benunit_id": [501], + "age": [80], + "employment_income": [0.0], + } + ), + ], + ignore_index=True, + ) + benunit = pd.concat( + [benunit, pd.DataFrame({"benunit_id": [501]})], ignore_index=True + ) + + result = build_uk_spi_support_channel( + person=person, + benunit=benunit, + household=household, + expected_spi_households=4, + seed=42, + source_year=2023, + strata_columns=("region",), + pension_age_share=0.2, + state_pension_age=66, + ) + + stacked = result.household.set_index( + ["source_household_id", support_channel_column("household")] + )["household_weight"] + # The premise: household 5's base row carries no weight, its copy some. + assert stacked[(5, BASE_FRS_SUPPORT_CHANNEL)] == 0.0 + assert stacked[(5, SPI_SYNTHETIC_SUPPORT_CHANNEL)] > 0.0 + assert result.pension_age_allocation["pension_age_households"] == 2 + assert result.pension_age_allocation["pension_age_spi_households"] == 3 + def test_build_spi_support_channel_without_a_pension_age_share_is_unchanged() -> None: person, benunit, household = _pension_age_frames() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py index b3070d785f..9a1a07127c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py @@ -1,8 +1,10 @@ from __future__ import annotations import json +import re import numpy as np +import pandas as pd import pytest from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate @@ -900,6 +902,187 @@ class the #787 review named. A non-exempt column now needs both bounds assert any("no clipped_high_rows allowance" in f for f in result.failures) +def _clip_evidence(low: int, high: int, **clip) -> dict: + """A was_wealth clip receipt of one column; ``clip`` adds to its block.""" + + return { + "stage": "was_wealth", + "support_clip": { + "columns": { + "cash_isa": { + "donor_min": 0.0, + "donor_max": 100.0, + "clipped_low_rows": low, + "clipped_high_rows": high, + "rows_considered": 30, + } + }, + **clip, + }, + } + + +def _clip_gate(evidence: dict, *, low: int = 1, high: int = 2): + return uk_stage_health_gate( + evidence=evidence, + stage="was_wealth", + check="support_clip", + parameters={ + "stage": "was_wealth", + "check": "support_clip", + "columns": ["cash_isa"], + "max_clipped_low_rows_by_column": {"cash_isa": low}, + "max_clipped_high_rows_by_column": {"cash_isa": high}, + }, + ) + + +def test_support_clip_gate_reads_one_clone_allowances_as_declared() -> None: + """A receipt without ``clone_count`` is one geographic clone: its + allowances, messages and details are as before (microcosm#931).""" + + passed = _clip_gate(_clip_evidence(1, 2)) + assert _passed(passed) + assert passed.details == { + "stage": "was_wealth", + "check": "support_clip", + "columns_checked": 1, + "exempt_columns": [], + } + failed = _clip_gate(_clip_evidence(2, 3)) + assert not _passed(failed) + assert list(failed.failures) == [ + "was_wealth: 'cash_isa' clipped_low_rows 2 exceeds 1.", + "was_wealth: 'cash_isa' clipped_high_rows 3 exceeds 2.", + ] + assert "clone_count" not in failed.details + # A receipt that says one clone reads as one without it. + assert _clip_gate(_clip_evidence(2, 3, clone_count=1)) == failed + + +def test_support_clip_gate_scales_its_allowances_by_the_clone_count() -> None: + """Every clone's rows are clipped, so ``K`` clones may clip ``K`` times the + declared rows, and no more (microcosm#931).""" + + passed = _clip_gate(_clip_evidence(3, 6, clone_count=3)) + assert _passed(passed) + assert passed.details["clone_count"] == 3 + failed = _clip_gate(_clip_evidence(4, 7, clone_count=3)) + assert not _passed(failed) + assert list(failed.failures) == [ + "was_wealth: 'cash_isa' clipped_low_rows 4 exceeds 3, 1 for each of the " + "3 geographic clones.", + "was_wealth: 'cash_isa' clipped_high_rows 7 exceeds 6, 2 for each of the " + "3 geographic clones.", + ] + assert failed.details["clone_count"] == 3 + # A zero allowance stays zero at any clone count. + assert not _passed(_clip_gate(_clip_evidence(1, 0, clone_count=3), low=0)) + assert _passed(_clip_gate(_clip_evidence(0, 0, clone_count=3), low=0, high=0)) + + +@pytest.mark.parametrize("clone_count", [0, -1, True, 2.0, "3", None]) +def test_support_clip_gate_refuses_a_malformed_clone_count(clone_count) -> None: + with pytest.raises(ValueError, match="clone_count must be a positive integer"): + _clip_gate(_clip_evidence(0, 0, clone_count=clone_count)) + + +def test_support_clip_receipt_gives_the_clone_count_beyond_one_clone() -> None: + """The shared clip receipt keeps its one-clone form and gives + ``clone_count`` only from two clones up (microcosm#931).""" + + from microcosm.build.uk_runtime.support_clip import ( + support_clip_to_donor_with_receipt, + ) + + draws = pd.DataFrame({"cash_isa": [-1.0, 50.0, 101.0]}) + donor = pd.DataFrame({"cash_isa": [0.0, 100.0]}) + + def clip(**keywords): + return support_clip_to_donor_with_receipt( + draws, donor, columns=["cash_isa"], stage="was_wealth", **keywords + ) + + column = { + "donor_min": 0.0, + "donor_max": 100.0, + "clipped_low_rows": 1, + "clipped_high_rows": 1, + "rows_considered": 3, + } + assert clip().receipt.evidence() == {"columns": {"cash_isa": column}} + assert clip(clone_count=1).receipt.evidence() == {"columns": {"cash_isa": column}} + cloned = clip(clone_count=3) + assert cloned.receipt.evidence() == { + "columns": {"cash_isa": column}, + "clone_count": 3, + } + pd.testing.assert_frame_equal(cloned.clipped, clip().clipped) + for malformed in (0, True, 2.5): + with pytest.raises(ValueError, match="clone_count must be a positive"): + clip(clone_count=malformed) + + +def test_the_support_clip_stages_read_the_shared_clone_count() -> None: + """The six support-clip stages read ``K`` through the geographic clones' + leaf (``geographic_clone_count``), which names the calling stage in its + refusals, as fixed sentences: never a count or an id (microcosm#931).""" + + from microcosm.build.uk_runtime import ( + etb_services, + etb_vat, + lcfs_consumption, + nts_bus_travel, + support_clip, + was_lisa, + was_wealth, + ) + from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count + + assert not hasattr(support_clip, "support_clip_clone_count") + for module in ( + etb_services, + etb_vat, + lcfs_consumption, + nts_bus_travel, + was_lisa, + was_wealth, + ): + assert module.geographic_clone_count is geographic_clone_count + plain = pd.DataFrame({"household_id": [11, 12]}) + assert geographic_clone_count(plain, stage="was_wealth") == 1 + cloned = pd.DataFrame( + { + "household_id": [11, 12, 1_000_000_011, 1_000_000_012, 2_000_000_011], + "household_clone_index": np.array([0, 0, 1, 1, 2], dtype=np.int64), + } + ) + assert geographic_clone_count(cloned, stage="was_wealth") == 3 + rule = ( + "was_wealth counts K geographic clones as the " + "'household_clone_index' values 0 to K - 1 (microcosm#931)." + ) + for index, fault in ( + ( + np.array([0, 2, 2], dtype=np.int64), + "An index from 0 to the largest is carried by no household.", + ), + ( + np.array([0, -1, -1], dtype=np.int64), + "A household carries a negative 'household_clone_index'.", + ), + ( + pd.array([0, None, None], dtype="Int64"), + "A household carries no 'household_clone_index'.", + ), + ): + frame = pd.DataFrame({"household_id": np.arange(11, 11 + len(index))}) + frame["household_clone_index"] = index + with pytest.raises(ValueError) as refused: + geographic_clone_count(frame, stage="was_wealth") + assert str(refused.value) == f"{fault} {rule}" + + def _wealth_coherence_evidence() -> dict: return { "stage": "was_wealth", @@ -2187,40 +2370,37 @@ def _spi_income_band_donor_evidence() -> dict[str, object]: } for lower, (count, taxpayers) in plan.items() ] + # Every funding row keeps its stratum and rounded donors; its masses and + # factor are withheld, and the two booleans state the funding + # (microcosm#931). return { "stage": "spi_income_band_donors", "minimum_donors_per_band": 120, "maximum_donor_weight": 250.0, + "maximum_reallocation_share": 0.5, "clone_count": 1, "seating_scale": "full", "mass_scale": 1.0, "sample_fraction": 1.0, "donor_count": 1_920, "reallocated_mass": 450_000.0, + "funding_within_share": True, + "funding_matches_bands": True, "mass": {"old_total": 28_600_000.0, "new_total": 28_600_000.0}, "bands": rows, "funding": [ { - "stratum": "LONDON", - "donor_households": 1_200, - "incumbent_mass": 3_700_000.0, - "donor_mass": 300_000.0, - "factor": 1.0 - 300_000.0 / 3_700_000.0, - }, - { - "stratum": "SOUTH_EAST", - "donor_households": 720, - "incumbent_mass": 3_900_000.0, - "donor_mass": 150_000.0, - "factor": 1.0 - 150_000.0 / 3_900_000.0, - }, - { - "stratum": "WALES", - "donor_households": 0, - "incumbent_mass": 1_400_000.0, - "donor_mass": 0.0, - "factor": 1.0, - }, + "stratum": stratum, + "donor_households": donors, + "incumbent_mass": "<3", + "donor_mass": "<3", + "factor": "<3", + } + for stratum, donors in ( + ("LONDON", 1_200), + ("SOUTH_EAST", 700), + ("WALES", 0), + ) ], } @@ -2235,29 +2415,44 @@ def _spi_income_band_donor_evidence() -> dict[str, object]: "maximum_relative_mass_deviation": 1e-9, "band_lower_bounds": [200_000, 500_000, 1_000_000, 2_000_000], } +_BAND_DONOR_BOUNDS = (200_000, 500_000, 1_000_000, 2_000_000) -def _spi_income_band_donor_gate(evidence: dict[str, object]): +def _spi_income_band_donor_gate(evidence: dict[str, object], frame=None): return uk_stage_health_gate( evidence=evidence, stage="spi_income_band_donors", check="spi_income_band_donor_support", parameters=_SPI_INCOME_BAND_DONOR_GATE_PARAMETERS, + frame=frame, ) +def _no_data_digit(message: str) -> bool: + """Whether ``message`` holds no digit beyond the declared band bounds.""" + + bounds = "|".join(str(band) for band in sorted(_BAND_DONOR_BOUNDS, reverse=True)) + return re.search(r"\d", re.sub(rf"\b(?:{bounds})\b", "", message)) is None + + def test_spi_income_band_donor_support_gate_checks_every_reserved_band() -> None: evidence = _spi_income_band_donor_evidence() result = _spi_income_band_donor_gate(evidence) assert _passed(result) assert result.details["donor_count"] == 1_920 assert result.details["heaviest_donor_weight"] == 250.0 + assert result.details["funding_within_share"] is True + assert "smallest_funding_factor" not in result.details rows = evidence["bands"] # A band whose copy lost its carrier fails. broken = [dict(row) for row in rows] broken[-1]["carriers"] = 119 result = _spi_income_band_donor_gate({**evidence, "bands": broken}) - assert not result.passed and "119 carriers" in " ".join(result.failures) + assert not result.passed + assert result.failures == ( + "spi_income_band_donors: band from 2000000's seats do not each hold one " + "carrier.", + ) # A missing band fails against the declared four. result = _spi_income_band_donor_gate({**evidence, "bands": rows[:-1]}) assert not result.passed and "differ from the declared" in " ".join(result.failures) @@ -2265,18 +2460,22 @@ def test_spi_income_band_donor_support_gate_checks_every_reserved_band() -> None off = [dict(row) for row in rows] off[0]["donor_weight"] = 1.0 result = _spi_income_band_donor_gate({**evidence, "bands": off}) - assert not result.passed and "differ from the published" in " ".join( + assert not result.passed and "differ from the published count" in " ".join( result.failures ) # A donor above the maximum weight fails, as does a band seated below the # minimum at full scale. heavy = [dict(row) for row in rows] heavy[0].update(donor_households=718, carriers=718, donor_weight=500.0) - failures = " ".join( - _spi_income_band_donor_gate({**evidence, "bands": heavy}).failures - ) - assert "exceeds the maximum 250.0" in failures - assert "the plan seats 1436" in failures + failures = _spi_income_band_donor_gate({**evidence, "bands": heavy}).failures + assert ( + "spi_income_band_donors: band from 200000 donor weight exceeds the maximum." + ) in failures + assert ( + "spi_income_band_donors: band from 200000 seats other than its plan at full " + "scale, or fewer than the minimum." + ) in failures + assert all(_no_data_digit(failure) for failure in failures) def test_spi_income_band_donor_support_gate_holds_the_mass_conserved() -> None: @@ -2288,77 +2487,226 @@ def test_spi_income_band_donor_support_gate_holds_the_mass_conserved() -> None: assert not result.passed and "funded from the incumbent households" in " ".join( result.failures ) - # A stratum scaled below the funding floor fails. - funding = [dict(row) for row in evidence["funding"]] - funding[0]["factor"] = 0.49 - result = _spi_income_band_donor_gate({**evidence, "funding": funding}) - assert not result.passed and "outside [0.5, 1]" in " ".join(result.failures) - # Funding rows that do not account for the reallocated mass fail. - funding = [dict(row) for row in evidence["funding"]] - funding[1]["donor_mass"] = 100_000.0 - result = _spi_income_band_donor_gate({**evidence, "funding": funding}) - assert not result.passed and "funding strata carry donor mass" in " ".join( - result.failures + # Bands that do not carry the reallocated mass fail. + result = _spi_income_band_donor_gate({**evidence, "reallocated_mass": 400_000.0}) + assert result.failures == ( + "spi_income_band_donors: the bands carry other donor mass than the receipt " + "reallocated.", ) # The receipt must be the reviewed rule's. result = _spi_income_band_donor_gate({**evidence, "maximum_donor_weight": 500.0}) - assert not result.passed and "differs from the gate's 250.0" in " ".join( - result.failures + assert result.failures == ( + "spi_income_band_donors: receipt maximum_donor_weight differs from the " + "gate's declared value.", ) with pytest.raises(ValueError, match="funding must be a non-empty list"): _spi_income_band_donor_gate({**evidence, "funding": []}) -def test_spi_income_band_donor_support_gate_scales_only_off_the_full_sample() -> None: +def test_spi_income_band_donor_support_gate_reads_the_funding_from_its_booleans() -> ( + None +): + """The funding rows withhold their masses and factors at every seating, so + the gate reads the funding from the receipt's two booleans and holds the + stage's declared share to its minimum funding factor (microcosm#931).""" + + evidence = _spi_income_band_donor_evidence() + messages: list[str] = [] + # A stratum that gave up more than the declared share, or funding that + # does not carry the bands' mass, is a false boolean. + for key, meaning in ( + ( + "funding_within_share", + "every funding stratum keeps at least one less the declared " + "reallocation share", + ), + ( + "funding_matches_bands", + "the bands and the funding strata each carry the reallocated mass", + ), + ): + result = _spi_income_band_donor_gate({**evidence, key: False}) + assert not result.passed + assert result.failures == ( + f"spi_income_band_donors: the receipt's {key} is not true: {meaning}.", + ) + assert result.details[key] is False + messages.extend(result.failures) + # A receipt without the boolean, or with another value, fails alike. + missing = {name: value for name, value in evidence.items() if name != key} + result = _spi_income_band_donor_gate(missing) + assert result.failures == ( + f"spi_income_band_donors: the receipt carries no {key}: {meaning}.", + ) + assert result.details[key] is None + result = _spi_income_band_donor_gate({**evidence, key: 1}) + assert not result.passed and result.details[key] is None + assert result.failures == ( + f"spi_income_band_donors: the receipt's {key} is not true: {meaning}.", + ) + messages.extend(result.failures) + # A stage share looser than the gate's minimum funding factor fails: one + # less it lets a stratum keep less than the minimum. + result = _spi_income_band_donor_gate( + {**evidence, "maximum_reallocation_share": 0.6} + ) + assert result.failures == ( + "spi_income_band_donors: the receipt's maximum_reallocation_share lets a " + "funding stratum keep less than the gate's minimum funding factor.", + ) + messages.extend(result.failures) + # A stricter share passes, and a share of one fails. + stricter = {**evidence, "maximum_reallocation_share": 0.25} + assert _passed(_spi_income_band_donor_gate(stricter)) + result = _spi_income_band_donor_gate( + {**evidence, "maximum_reallocation_share": 1.0} + ) + assert result.failures == ( + "spi_income_band_donors: the receipt's maximum_reallocation_share is not " + "a share strictly between zero and one.", + ) + messages.extend(result.failures) + assert all(_no_data_digit(message) for message in messages) + + +class _BandDonorFrame: + """A synthetic transferred-phase frame holding a band donor layer. + + Each declared band seats ``seats`` households in each of ``clone_count`` + clones, each id in the band donors' copy-id ordinal and the clone's, with + one carrier and one other member; two incumbent households hold no + seat, and a later layer's copy of the first seat (the CGT support + split's ordinal one) carries the donor flag and a carrier too. Only + ``table`` is read, as the gate reads a frame. + """ + + def __init__(self, *, seats: int = 2, clone_count: int = 1) -> None: + household_ids: list[int] = [11, 12] + bands: list[float] = [0.0, 0.0] + clones: list[int] = [0, 0] + donor: list[bool] = [False, False] + for clone in range(clone_count): + for position, band in enumerate(_BAND_DONOR_BOUNDS): + for seat in range(seats): + household_ids.append( + 2 * 10**11 + clone * 10**9 + 100 * position + seat + 21 + ) + bands.append(float(band)) + clones.append(clone) + donor.append(True) + later = household_ids[2] + 10**13 + household_ids.append(later) + bands.append(bands[2]) + clones.append(clones[2]) + donor.append(True) + self.household = pd.DataFrame( + { + "household_id": np.asarray(household_ids, dtype=np.int64), + "household_is_spi_income_band_donor": donor, + "spi_income_band_donor_lower_bound": bands, + "household_clone_index": np.asarray(clones, dtype=np.int64), + } + ) + people = [ + (household_id, carrier) + for household_id, is_donor in zip(household_ids, donor, strict=True) + for carrier in (is_donor, False) + ] + self.person = pd.DataFrame( + { + "person_household_id": np.asarray( + [household_id for household_id, _ in people], dtype=np.int64 + ), + "person_is_spi_income_band_carrier": [carrier for _, carrier in people], + } + ) + + def table(self, entity: str) -> pd.DataFrame: + return {"household": self.household, "person": self.person}[entity] + + +def _capped_band_donor_evidence(*, clone_count: int = 1) -> dict[str, object]: + """A capped receipt: rounded counts in source households, the masses + withheld, the band rows without their carriers' regions.""" + evidence = _spi_income_band_donor_evidence() - # Given a small frame: fewer, lighter donors. The bands fall short of the - # published mass by construction and the gate passes on the receipt. scale = 0.5 rows = [ { - **row, - "donor_households": 2, - "carriers": 2, - "donor_weight": row["planned_donor_weight"] * scale, + "lower_bound": row["lower_bound"], + "expected_donor_households": clone_count * row["expected_donor_households"], + "donor_households": "<3", + "carriers": "<3", + "eligible_households": 20, + "planned_donor_weight": row["planned_donor_weight"] / clone_count, + "donor_weight": row["planned_donor_weight"] * scale / clone_count, + "weighted_taxpayers": "<3", + "published_taxpayers": row["published_taxpayers"], + "carrier_mean_age": "<3", } for row in evidence["bands"] ] - donor_mass = sum(row["donor_weight"] * 2 for row in rows) - scaled = { + return { **evidence, + "clone_count": clone_count, "seating_scale": "capped", "mass_scale": scale, "sample_fraction": 0.01, - "donor_count": 8, - "reallocated_mass": donor_mass, + "donor_count": "3-9", + "reallocated_mass": "<3", "bands": rows, "funding": [ { "stratum": "LONDON", - "donor_households": 8, - "incumbent_mass": 2.0 * donor_mass, - "donor_mass": donor_mass, - "factor": 0.5, + "donor_households": "3-9", + "incumbent_mass": "<3", + "donor_mass": "<3", + "factor": "<3", } ], } - assert _passed(_spi_income_band_donor_gate(scaled)) + + +def test_spi_income_band_donor_support_gate_scales_only_off_the_full_sample() -> None: + # Given a small frame: fewer, lighter donors. The bands fall short of the + # published mass by construction; the gate recomputes the capped seats + # from the frame and passes. + scaled = _capped_band_donor_evidence() + frame = _BandDonorFrame() + result = _spi_income_band_donor_gate(scaled, frame) + assert _passed(result), result.failures + # Its details carry the clone count, the seating, the mass scale, the + # booleans and the count of bands, and nothing the frame counts. + assert result.details == { + "stage": "spi_income_band_donors", + "check": "spi_income_band_donor_support", + "bands_checked": 4, + "clone_count": 1, + "seating_scale": "capped", + "mass_scale": 0.5, + "funding_within_share": True, + "funding_matches_bands": True, + } + messages: list[str] = [] # The same receipt on a full-sample build fails: a full build seats and # funds the donors at full scale. - result = _spi_income_band_donor_gate({**scaled, "sample_fraction": 1.0}) - assert not result.passed and "at full scale" in " ".join(result.failures) - assert ( + result = _spi_income_band_donor_gate({**scaled, "sample_fraction": 1.0}, frame) + assert result.failures == ( "spi_income_band_donors: a full-sample build must seat and fund the donors " - "at full scale, got seating_scale capped and mass_scale 0.5." - ) in result.failures - result = _spi_income_band_donor_gate({**scaled, "mass_scale": 0.0}) - assert not result.passed and "outside (0, 1]" in " ".join(result.failures) + "at full scale, got seating_scale capped and a mass_scale other than one.", + ) + messages.extend(result.failures) + result = _spi_income_band_donor_gate({**scaled, "mass_scale": 0.0}, frame) + assert result.failures == ( + "spi_income_band_donors: mass_scale is not a share above zero and at most one.", + ) + messages.extend(result.failures) # The receipt records the seating scale as full or capped, never as the # exact scale, which gives back the eligible household count. # A value in neither form is named by its type alone, so a replayed old # receipt's exact scale does not reach the gate's verdict. result = _spi_income_band_donor_gate( - {**scaled, "seating_scale": 8 / 1_920, "sample_fraction": 1.0} + {**scaled, "seating_scale": 8 / 1_920, "sample_fraction": 1.0}, frame ) assert not result.passed assert ( @@ -2366,16 +2714,126 @@ def test_spi_income_band_donor_support_gate_scales_only_off_the_full_sample() -> ) in result.failures assert ( "spi_income_band_donors: a full-sample build must seat and fund the donors " - "at full scale, got seating_scale a float and mass_scale 0.5." + "at full scale, got seating_scale a float and a mass_scale other than one." ) in result.failures assert result.details["seating_scale"] == "a float" assert not any("0.0041" in failure for failure in result.failures) + messages.extend(result.failures) result = _spi_income_band_donor_gate( - {key: value for key, value in scaled.items() if key != "seating_scale"} + {key: value for key, value in scaled.items() if key != "seating_scale"}, frame ) assert ( "spi_income_band_donors: seating_scale is missing, neither 'full' nor 'capped'." ) in result.failures + messages.extend(result.failures) + # Without the frame, a capped receipt is not passed. + result = _spi_income_band_donor_gate(scaled) + assert result.failures == ( + "spi_income_band_donors: a capped seating's seats are recomputed from the " + "phase's frame, and the gate was given none.", + ) + messages.extend(result.failures) + assert all(_no_data_digit(message) for message in messages) + + +def test_spi_income_band_donor_support_gate_recomputes_a_capped_seating() -> None: + """A capped seating's seats are the band donor rows that are originals in + every copy-id field minted after the band donors; each holds one carrier, + and each band's seats split evenly across the clones (microcosm#931).""" + + evidence = _capped_band_donor_evidence(clone_count=3) + frame = _BandDonorFrame(clone_count=3) + assert _passed(_spi_income_band_donor_gate(evidence, frame)) + household, person = frame.household, frame.person + later = household["household_id"].ge(10**13) + first_seat = int(household.loc[2, "household_id"]) + messages: list[str] = [] + + def gate_on(*, household=household, person=person): + broken = _BandDonorFrame(clone_count=3) + broken.household, broken.person = household, person + return _spi_income_band_donor_gate(evidence, broken) + + # Read as a seat, the later copy of the first seat would split its band + # unevenly; the gate reads it as the copy it is. + assert household.loc[later, "household_is_spi_income_band_donor"].all() + # A seat whose carrier lost its flag holds no carrier, and a seat with a + # second carrier holds two. + for flags in ( + person["person_household_id"].eq(first_seat) + & person["person_is_spi_income_band_carrier"], + person["person_household_id"].eq(first_seat) + & ~person["person_is_spi_income_band_carrier"], + ): + flipped = person.copy() + flipped.loc[flags, "person_is_spi_income_band_carrier"] = ~flipped.loc[ + flags, "person_is_spi_income_band_carrier" + ] + result = gate_on(person=flipped) + assert result.failures == ( + "spi_income_band_donors: band from 200000's seats do not each hold " + "one carrier.", + ) + messages.extend(result.failures) + # A seat of clone 1 that is no longer a donor leaves its band uneven. + seat = household["household_clone_index"].eq(1) & household[ + "spi_income_band_donor_lower_bound" + ].eq(500_000.0) + unseated = household.copy() + unseated.loc[seat.idxmax(), "household_is_spi_income_band_donor"] = False + result = gate_on(household=unseated) + assert result.failures == ( + "spi_income_band_donors: band from 500000's donors do not split evenly " + "across the geographic clones.", + ) + messages.extend(result.failures) + # A band without a seat, or a seat in an undeclared band, fails. + empty = household.copy() + empty.loc[ + empty["spi_income_band_donor_lower_bound"].eq(2_000_000.0), + "household_is_spi_income_band_donor", + ] = False + result = gate_on(household=empty) + assert result.failures == ( + "spi_income_band_donors: band from 2000000 seats no donor.", + ) + messages.extend(result.failures) + undeclared = household.copy() + undeclared.loc[2, "spi_income_band_donor_lower_bound"] = 150_000.0 + result = gate_on(household=undeclared) + assert "spi_income_band_donors: a seat holds a band the gate does not declare." in ( + result.failures + ) + messages.extend(result.failures) + # A frame without the band donor columns, or with ids that are not + # non-negative integers, cannot be read. + result = gate_on( + household=household.drop(columns=["household_is_spi_income_band_donor"]) + ) + assert result.failures == ( + "spi_income_band_donors: the frame carries no " + "household.household_is_spi_income_band_donor, so the capped seating's " + "seats cannot be recomputed.", + ) + messages.extend(result.failures) + result = gate_on( + household=household.assign(household_id=household["household_id"] * 1.0) + ) + assert result.failures == ( + "spi_income_band_donors: the frame's household ids are not non-negative " + "integers, so the capped seating's seats cannot be recomputed.", + ) + messages.extend(result.failures) + # Read against two clones, the three clones' seats do not split. + result = _spi_income_band_donor_gate({**evidence, "clone_count": 2}, frame) + assert not result.passed + assert set(result.failures) == { + f"spi_income_band_donors: band from {band}'s donors do not split evenly " + "across the geographic clones." + for band in _BAND_DONOR_BOUNDS + } + messages.extend(result.failures) + assert all(_no_data_digit(message) for message in messages) def _three_clone_band_donor_evidence() -> dict[str, object]: @@ -2408,30 +2866,35 @@ def test_spi_income_band_donor_support_gate_scales_its_checks_by_the_clone_count assert _passed(result), result.failures assert result.details["clone_count"] == 3 assert result.details["heaviest_donor_weight"] == pytest.approx(250.0 / 3) + messages: list[str] = [] # A donor whose weight times the clone count exceeds the maximum fails. heavy = [dict(row) for row in evidence["bands"]] heavy[0]["donor_weight"] = 100.0 + failures = _spi_income_band_donor_gate({**evidence, "bands": heavy}).failures assert ( - "spi_income_band_donors: band from 200000 donor weight 100.0 exceeds the " - "maximum 250.0 divided by the geographic clone count 3." - ) in _spi_income_band_donor_gate({**evidence, "bands": heavy}).failures + "spi_income_band_donors: band from 200000 donor weight exceeds the " + "maximum divided by the geographic clone count." + ) in failures + messages.extend(failures) # A band whose donors do not split evenly across the clones fails, as does # one seated below three clones' minimum at full scale. uneven = [dict(row) for row in evidence["bands"]] uneven[1].update(donor_households=733, carriers=733) failures = _spi_income_band_donor_gate({**evidence, "bands": uneven}).failures assert ( - "spi_income_band_donors: band from 500000 seats 733 donors, which do not " - "split evenly across the 3 geographic clones." + "spi_income_band_donors: band from 500000's donors do not split evenly " + "across the geographic clones." ) in failures + messages.extend(failures) short = [dict(row) for row in evidence["bands"]] short[3].update(expected_donor_households=357, donor_households=357, carriers=357) + failures = _spi_income_band_donor_gate({**evidence, "bands": short}).failures assert ( - "spi_income_band_donors: band from 2000000 seats 357 donors at full scale; " - "the plan seats 357 and the minimum is 360 (120 in each of the 3 " - "geographic clones)." - ) in _spi_income_band_donor_gate({**evidence, "bands": short}).failures + "spi_income_band_donors: band from 2000000 seats other than its plan at " + "full scale, or fewer than the minimum in each geographic clone." + ) in failures + messages.extend(failures) # The same rows read as one clone's: no row is checked against three # clones, and the one-clone messages are as they were. one = _spi_income_band_donor_evidence() @@ -2439,13 +2902,14 @@ def test_spi_income_band_donor_support_gate_scales_its_checks_by_the_clone_count heavy[0].update(donor_households=718, carriers=718, donor_weight=500.0) failures = _spi_income_band_donor_gate({**one, "bands": heavy}).failures assert ( - "spi_income_band_donors: band from 200000 donor weight 500.0 exceeds the " - "maximum 250.0." + "spi_income_band_donors: band from 200000 donor weight exceeds the maximum." ) in failures assert ( - "spi_income_band_donors: band from 200000 seats 718 donors at full scale; " - "the plan seats 1436 and the minimum is 120." + "spi_income_band_donors: band from 200000 seats other than its plan at full " + "scale, or fewer than the minimum." ) in failures + messages.extend(failures) + assert all(_no_data_digit(message) for message in messages) # The clone count must be a positive integer the receipt states. for value in (None, 0, -1, 1.0, True, "3"): @@ -2987,6 +3451,58 @@ def test_property_components_gate_caps_a_reported_allowance_at_two_weights() -> assert failed.details["band_errors"][2]["allowance_weight"] == 19_000.0 +def _property_rows_gate(below: int, above: int, *, clone_count=None, maximum=1): + evidence = _property_components_evidence() + evidence["receipts"].update( + rows_receipts_below_profit=below, rows_with_profit_above_their_band=above + ) + if clone_count is not None: + evidence["clone_count"] = clone_count + return uk_stage_health_gate( + evidence=evidence, + stage="property_components", + check="property_components", + parameters={ + **_PROPERTY_COMPONENTS_PARAMETERS, + "maximum_rows_receipts_below_profit": maximum, + "maximum_rows_with_profit_above_band": maximum, + }, + ) + + +def test_property_components_gate_scales_its_row_maxima_by_the_clone_count() -> None: + """The two row maxima are declared for one geographic clone: a receipt + without ``clone_count`` reads them as declared, and ``K`` clones may + carry ``K`` times the rows (microcosm#931).""" + + one = _property_rows_gate(1, 1) + assert _passed(one) + assert "clone_count" not in one.details + over = _property_rows_gate(2, 2) + assert list(over.failures) == [ + "property_components: 2 rows carry receipts below profit (allowed 1).", + "property_components: 2 landlords sit in a receipts band below their " + "profit (allowed 1).", + ] + assert _property_rows_gate(2, 2, clone_count=1) == over + + cloned = _property_rows_gate(3, 3, clone_count=3) + assert _passed(cloned) + assert cloned.details["clone_count"] == 3 + failed = _property_rows_gate(4, 4, clone_count=3) + assert list(failed.failures) == [ + "property_components: 4 rows carry receipts below profit (allowed 3, 1 " + "for each of the 3 geographic clones).", + "property_components: 4 landlords sit in a receipts band below their " + "profit (allowed 3, 1 for each of the 3 geographic clones).", + ] + # The declared zeros stay zero at any clone count. + assert not _passed(_property_rows_gate(0, 1, clone_count=3, maximum=0)) + assert _passed(_property_rows_gate(0, 0, clone_count=3, maximum=0)) + with pytest.raises(ValueError, match="clone_count must be a positive integer"): + _property_rows_gate(0, 0, clone_count=0) + + def test_consumption_basis_gate_checks_the_recorded_bridge_factor() -> None: # The receipt records before, after, base and factor for each bridged # column; the gate holds factor to after / base (the #1167 re-review's nit). @@ -3213,6 +3729,22 @@ def test_regional_home_values_gate_holds_the_ranking_contract(mutate, fragment) assert any(fragment in failure for failure in result.failures), result.failures +def test_regional_home_values_gate_holds_one_cap_limit_at_any_clone_count() -> None: + """The stage counts the sources at the value cap in geographic clone 0's + rows (test_uk_regional_home_values), so the check has no clone + allowance: a clone count in the receipt does not widen its limit + (microcosm#931).""" + + gate_id = "uk_stage_regional_home_values_ranking" + for at_cap in (1, "<3", "3-9"): + receipt = _home_values_receipt() + receipt["sources_at_cap"] = at_cap + one = _gate(gate_id, receipt) + cloned = _gate(gate_id, {**receipt, "clone_count": 3}) + assert not one.passed + assert cloned == one + + def test_regional_home_values_gate_flags_a_region_of_fewer_than_three_rows() -> None: """A region the receipt prints "<3" rewrote one or two rows: not empty, and flagged rather than failed (María's ruling of 2026-10-26).""" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_take_up_gate.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_take_up_gate.py index c9c52b8043..e07f8f0703 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_take_up_gate.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_take_up_gate.py @@ -326,6 +326,78 @@ def _stage_gate(stage: str, receipt: dict[str, object]): ) +def test_stage_receipt_counts_support_and_values_in_clone_zero() -> None: + """Each geographic clone draws its own flags, so the receipt counts a + flag's population support and its distinct values in clone 0's rows, as + one clone counts them, while the shares are measured over every clone + (microcosm#931).""" + + from microcosm.build.uk_runtime.frs_take_up import ( + UK_EXTENDED_CHILDCARE_OUTPUT, + geographic_clone_zero_rows, + ) + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + one = {row["output"]: row for row in _receipt(_frame(), "frs_take_up")["flags"]} + frame = clone_first_frame(_frame(), 3) + benunit = frame.table("benunit") + clone = benunit["benunit_id"].to_numpy() // 10**9 + # Clones 1 and 2 claim the working-parent hours in every unit, so their + # under-2s join the usage population in households clone 0 leaves out. + benunit.loc[clone > 0, UK_EXTENDED_CHILDCARE_OUTPUT] = True + # Clone 0 claims Tax-Free Childcare in every unit; the others alternate. + benunit.loc[clone == 0, "would_claim_tfc"] = True + rows = {row["output"]: row for row in _receipt(frame, "frs_take_up")["flags"]} + + usage = rows["person.max_free_entitlement_hours_used"] + assert ( + usage["population_support"] + == one["person.max_free_entitlement_hours_used"]["population_support"] + ) + assert usage["weighted_share"] == "<3" + assert rows["benunit.would_claim_tfc"]["distinct_values"] == 1 + assert ( + rows["benunit.would_claim_tfc"]["population_support"] + == one["benunit.would_claim_tfc"]["population_support"] + ) + failures = _stage_gate("frs_take_up", _receipt(frame, "frs_take_up")).failures + assert any("would_claim_tfc is constant" in failure for failure in failures) + # Each row reads its household's clone: people, units and households. + for entity, ids in ( + ("household", "household_id"), + ("benunit", "benunit_id"), + ("person", "person_id"), + ): + zero = geographic_clone_zero_rows(frame, entity) + assert (zero == (frame.table(entity)[ids].to_numpy() // 10**9 == 0)).all() + assert geographic_clone_zero_rows(_frame(), entity).all() + + +def test_stage_receipt_withholds_a_share_other_clones_carry_on_one_household() -> None: + """The share and the target are measured over every clone, so they are + withheld where one or two source households carry the population over + every clone, though clone 0 carries none of it and its printed support + is 0 (microcosm#931).""" + + from microcosm.build.uk_runtime.frs_take_up import UK_EXTENDED_CHILDCARE_OUTPUT + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + frame = clone_first_frame(_frame(), 3) + benunit = frame.table("benunit") + # Only clone 1's copy of unit 201 claims the working-parent hours, so the + # under-2 usage population is one child of one source household, none of + # them in clone 0. + benunit[UK_EXTENDED_CHILDCARE_OUTPUT] = benunit["benunit_id"].eq(201 + 10**9) + usage = {row["output"]: row for row in _receipt(frame, "frs_take_up")["flags"]}[ + "person.max_free_entitlement_hours_used" + ] + + assert "unmeasured" not in usage + assert usage["population_support"] == 0 + assert usage["weighted_share"] == "<3" + assert usage["target"] == "<3" + + def test_stage_receipt_measures_drawn_flags_and_withholds_thin_populations() -> None: receipt = _receipt(_frame(), "frs_take_up") rows = {row["output"]: row for row in receipt["flags"]} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py index 4210b68d3b..00b0a3a012 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_capital_coherence.py @@ -772,3 +772,33 @@ def test_a_high_value_copy_counts_as_a_donor_of_its_own() -> None: ) assert alone.coarsening_levels["exact_cell"] == 2 assert copied.coarsening_levels["exact_cell"] == 3 + + +def test_the_capital_fold_receipt_is_absent_at_one_clone_and_rounded_at_three() -> None: + """The stage records how its donor fold met its survey records only on a + frame of more than one geographic clone: the clone count, the selected + donors' and the varied donors' distinct source households, rounded, and + the varied share only from ten varied households up (microcosm#931).""" + + from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_FOLD_EVIDENCE + + for frame in (_frame(), clone_first_frame(_frame(), 1)): + result = cohere_uc_capital(frame, minimum_cell_donors=2) + assert result.geographic_fold is None + assert GEOGRAPHIC_FOLD_EVIDENCE not in result.evidence() + # Five base units donate, and no unit's clones differ. + alike = cohere_uc_capital(clone_first_frame(_frame(), 3), minimum_cell_donors=2) + assert alike.evidence()[GEOGRAPHIC_FOLD_EVIDENCE] == { + "clone_count": 3, + "records": "3-9", + "varied_records": 0, + "varied_share": 0, + } + # Every base unit's clones differ on the capital it donates. + varied = cohere_uc_capital(_variant_clones(_frame(), 3), minimum_cell_donors=2) + assert varied.evidence()[GEOGRAPHIC_FOLD_EVIDENCE] == { + "clone_count": 3, + "records": "3-9", + "varied_records": "3-9", + "varied_share": "3-9", + } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_reporter_redraw.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_reporter_redraw.py index 98765b7b8b..c3503cc5b0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_reporter_redraw.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_uc_reporter_redraw.py @@ -1,6 +1,9 @@ """Tests split from packages/microcosm-build/tests/test_uk_uc_reporter_redraw.py.""" # ruff: noqa: F403, F405 +import json + +from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_FOLD_EVIDENCE from microcosm.build.uk_runtime.national_frame import uk_national_frame from test_support.microcosm_build.uk_geographic_clones import with_geographic_clones from test_support.microcosm_build.uk_uc_reporter_redraw import * @@ -527,3 +530,71 @@ def test_clones_that_differ_on_a_predictor_train_as_variants_at_their_share() -> zip(table.loc[unit, "benunit_investment_income"], weights[unit], strict=True) ) == [(200.0, 1.0), (700.0, 1.0)] assert sorted(weights[~unit].tolist()) == [3.0, 4.0, 5.0] + + +def test_the_forests_fold_receipt_is_absent_at_one_clone_and_rounded_at_three() -> None: + """The stage records how its forest's geographic fold met the screened + benefit units only on more than one clone, so a one-clone frame's + evidence keeps its bytes; the counts are true source households, rounded + (microcosm#931).""" + + alone = redraw_spi_reported_uc( + _frame(), engine=_StubEngine(), qrf_factory=_StubQRFFactory() + ) + assert GEOGRAPHIC_FOLD_EVIDENCE not in alone.evidence() + three = redraw_spi_reported_uc( + with_geographic_clones(_frame(), n_clones=3), + engine=_StubEngine(), + qrf_factory=_StubQRFFactory(), + ) + # Four screened FRS units in four households train the forest. + assert three.evidence()[GEOGRAPHIC_FOLD_EVIDENCE] == { + "clone_count": 3, + "records": "3-9", + "varied_records": 0, + "varied_share": 0, + } + + +def _with_landlords(pool, person_ids: tuple[int, ...]): + """``pool`` with more property income on each of ``person_ids``.""" + + person = pool.table("person").copy() + for person_id in person_ids: + rows = person["person_id"] == person_id + assert rows.sum() == 1 + person.loc[rows, "property_income"] += 500.0 + return uk_national_frame( + person=person, + benunit=pool.table("benunit"), + household=pool.table("household"), + household_weights=pool.weights_for("household").values, + weight_kind=pool.weights_for("household").kind, + time_period="2024", + ) + + +def test_one_and_two_varied_units_give_the_same_fold_receipt_bytes() -> None: + """Clones drawn apart in one training unit's household and in two read + "<3", count and share alike, so the two receipts are the same bytes + (microcosm#931).""" + + pool = with_geographic_clones(_frame()) + receipts = [] + # Clone 1 of units 101 and 102's first members: ids move by 10,000 per + # clone. + for landlords in ((11011,), (11011, 11021)): + result = redraw_spi_reported_uc( + _with_landlords(pool, landlords), + engine=_StubEngine(), + qrf_factory=_StubQRFFactory(), + ) + receipt = result.evidence()[GEOGRAPHIC_FOLD_EVIDENCE] + receipts.append(json.dumps(receipt, sort_keys=True)) + assert receipts[0] == receipts[1] + assert json.loads(receipts[0]) == { + "clone_count": 2, + "records": "3-9", + "varied_records": "<3", + "varied_share": "<3", + } diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py index c7cd0485f3..ff3a31a0ed 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_lisa.py @@ -573,6 +573,41 @@ def _transform() -> UKWASLISAStageTransform: ) +def test_the_support_clip_receipt_gives_the_frames_clone_count() -> None: + """Every geographic clone's balances are clipped, so the clip receipt + gives the frame's clone count from two clones up, and the support-clip + check allows that multiple of its row allowances (microcosm#931).""" + + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + def clip(frame) -> dict: + transform = _transform() + transform(frame) + return transform.checkpoint_metadata()["evidence"]["support_clip"] + + one = clip(_recipient_frame()) + assert set(one) == {"columns"} + cloned = clip(clone_first_frame(_recipient_frame(), 3)) + assert cloned["clone_count"] == 3 + assert ( + cloned["columns"][BALANCE_COLUMN]["rows_considered"] + == 3 * one["columns"][BALANCE_COLUMN]["rows_considered"] + ) + allowances = { + "columns": [BALANCE_COLUMN], + "max_clipped_low_rows_by_column": {BALANCE_COLUMN: 0}, + "max_clipped_high_rows_by_column": {BALANCE_COLUMN: 0}, + } + gate = uk_stage_health_gate( + evidence={"stage": UK_WAS_LISA_STAGE_NAME, "support_clip": cloned}, + stage=UK_WAS_LISA_STAGE_NAME, + check="support_clip", + parameters=allowances, + ) + assert gate.passed, gate.failures + assert gate.details["clone_count"] == 3 + + def test_stage_writes_the_declared_outputs_with_receipts() -> None: frame = _recipient_frame() transform = _transform() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py index c368d9f54f..7aaeda979c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_was_wealth.py @@ -554,6 +554,44 @@ def test_student_loan_waterfall_exercises_equal_split_tiers() -> None: assert allocated.tolist() == [100.0, 0.0, 80.0, 0.0, 30.0, 30.0] +def test_the_support_clip_receipt_gives_the_frames_clone_count( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Every geographic clone's draws are clipped, so the clip receipt gives + the frame's clone count from two clones up, for the support-clip check to + multiply its row allowances by; a one-clone receipt keeps its form + (microcosm#931).""" + + import microcosm.build.uk_runtime.was_wealth as module + from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame + + donor = clean_was_household_table(_raw_was()) + + def fake_impute(*args, **kwargs): + rows = np.arange(len(kwargs["record_ids"])) % len(donor) + draws = donor.loc[:, _DRAW_COLUMNS].iloc[rows].reset_index(drop=True) + return module.UKWASWealthImputationResult(draws=draws, fit_weight_records=()) + + monkeypatch.setattr(module, "impute_was_wealth", fake_impute) + + def clip(frame) -> dict: + transform = UKWASWealthStageTransform( + stage=_stage(), engine=_FakeEngine(), donor=_raw_was() + ) + transform(frame) + return transform.checkpoint_metadata()["evidence"]["support_clip"] + + one = clip(_frame()) + assert set(one) == {"columns"} + cloned = clip(clone_first_frame(_frame(), 3)) + assert cloned["clone_count"] == 3 + assert set(cloned) == {"columns", "clone_count"} + assert all( + receipt["rows_considered"] == 3 * one["columns"][column]["rows_considered"] + for column, receipt in cloned["columns"].items() + ) + + def test_support_clip_and_integer_vehicle_output( monkeypatch: pytest.MonkeyPatch, ) -> None: From 330557f1376702e2a3b35a7203100eac657d5d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:38:43 +0000 Subject: [PATCH 061/108] Take the clone count on the spine CLI and hand it to the engine (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UK spine build takes its geographic clone count K: --n-clones, a whole number from 1 to 100, one by default. K goes to the geographic_expand node, the transform that makes the clones and the rules engine. Every engine call after geographic_expand runs one simulation per clone, and the engine refuses a frame that carries another count. The engine identity does not take K. Without the flag the spine builds one clone, as before. This follows María's rulings of 10 Nov: - decision 4: no national or dense build between groups (b) and (c), and both refuse a clone-indexed spine early. The dense expand has refused one since geographic_expand landed. The national line now refuses one at its first node that reads the frame; - the E4/E5 scope change rides this commit, in the identity tool. E4 and E5 compare with the store only what their own stages last wrote, on the inputs those stages read. Spine CLI (uk_runtime.spine_build): - --n-clones outside 1 to GEOGRAPHIC_CLONE_LIMIT, the geographic_support copy-id field's bound, is refused in a fixed sentence. K reaches uk_spine_graph(geographic_clones=K), UKGeographicExpandStageTransform(clones=K) and _rules_engine(clones=K). _rules_engine_provenance takes no clone count. - A release candidate takes one clone until group (c) pins the clone count a release builds at. The parser refuses --release-candidate with any other K. So does _validate_args, because the full build marks a spine request a release candidate after parsing it. - A --synthetic-fixture-dir whose config.geographic_clones differs from --n-clones is refused while the spine is prepared, before any stage runs, instead of at geographic_expand. - The build sidecar records geographic_clone_count, read from the final frame through geographic_clones.geographic_clone_count. A spine of more than one clone also binds K into the Logbook identity digest (_logbook_run_config); a one-clone build's digest is unchanged. - --stage-profile-dir is the memory guard ruled on 6 Nov. It runs every spine kernel, gates included, inside stage_profile.profile_stage under its node id, through a private proxy. The proxy forwards everything but run to the kernel it wraps and holds that kernel's bound implementation_hash. So node keys do not move, and a profiled run reuses an unprofiled run's store. The proxy declares no ref, so the census of kernel classes does not count it. - A profile record keeps the message of a failure its node raised, so the directory stays private. Before any node runs, _validate_args refuses, in fixed sentences, a profile directory inside the repository, a path that is not a directory, and a stage_profile.json there that is not a stage profile of the current schema, which the profiler would refuse only after the first node had run. stage_profile.json joins the output path-collision check, and .gitignore ignores it and its lock file. National line (uk_runtime.graph_national): - refuse_clone_first_national_spine refuses a bound spine whose households carry household_clone_index, at one clone too, in one fixed sentence naming the column and group (c). The national problem, the first national node given the bound frame, calls it before it resolves anything. Without it, a national build on a one-clone spine would solve in full and then fail its terminal degenerate-surface gate on the constant indices. Identity tool (tools/verify_uk_identity_stability.py): - --spine-sidecar takes the spine's build sidecar. The tool refuses a frame whose clone count differs from the sidecar's geographic_clone_count, or from the count cgt_support_split's receipt counted its copies on; the refusal gives only the clone counts. It refuses in fixed sentences a sidecar without those records and one whose output digest is not --input-h5's. Every receipt records the counts under geographic_clones. - E4's engine is PolicyEngineUKEngine(clones=K). - For each recomputed column, E4 and E5 read from the packaged source-stage manifest which later stages declare writing it. They report such a column under rewritten_later, with those stages, instead of comparing it with the store. A rewrite the receipt's scope takes back does not count: salary_sacrifice and regional_home_values, where the artifact keeps the column each reversal reads. E5 reports student_loan_balance (student_loans). E4 reports would_claim_uc, would_claim_pc, would_claim_child_benefit and child_benefit_opts_out, and the six columns childcare_copy_take_up declares rewriting, which by its notes it re-draws on copies only. So E4 also names, under rewritten_later_still_equal, the rewritten columns the store still holds bitwise. - E4 recomputes on what its stages read: - frs_brma's LHA_category is materialized on the frame that stage read, as far as the artifact restores it. Columns first written after frs_brma are dropped, and the flags frs_take_up drew that later stages rewrite are put back from the recompute. lha_category_scope names what stays as stored, with its writers, salary_sacrifice among them, whose reversal takes back only the records it converted. - The household weights are taken back, to rounding, through the band donors' funding factors and the SPI support channel's declared prior-mass shares (_frame_as_draw_stages_saw), which refuses a non-numeric age among the rows it keeps in a fixed sentence. is_apprentice is held exactly, or, where the restored weights move the walk by rounding, to the walk on them; held_to_tolerance then names it. - E8's band donor block gives the layer in the forms of the stage's capped receipt, at every seating: the donors and each band's donors in rounded distinct source households; the stage's funding rows, with their masses and factor withheld; and funding_within_share, the stage's funding rule recomputed from the stored weights, which E8 also checks (band_donor_funding_share). E8 holds each rebuilt factor at its own weight tolerance, a relative 1e-9 (_BAND_DONOR_FUNDING_RTOL), not the stage gate's few ulps: the stage's exact-total correction on one incumbent row moves a small stratum's rebuilt factor by more than that, so the stage check's tolerance would mark the stage's own output red. E8 no longer writes funding factors, band weights, weighted taxpayers or the mass scale. Its checks still compare them. spi_income_band_donors: its three clone-index refusals are fixed sentences naming the column, with no count. Text: changelog.d/931-uk-spine-clone-count.added.md is new. The unreleased band donor fragment describes the count-free refusals, and four other #931 fragments now say a spine is one clone by default. What moves: - At K = 1 no spine value and no gate verdict moves. The sidecar gains geographic_clone_count. A national build on any spine built since geographic_expand landed is refused at the national problem. - Node keys only. Of the 101 kernels the UK registries register, 6 move their kernel.implementation_hash() against the parent: spi_income_band_donors and hmrc_spi_income_spine, which hash spi_band_donors; and the national targets, problem, gates and readback, which hash graph_national. Every spine node from spi_income_band_donors on re-keys, and so does every national node but the bound checkpoint. The driver's own changes move no kernel hash, and the profile proxy keys as the kernel it wraps. - No declaration, generated file, fixture or gates.json changes, and H2 is not regenerated. No module outside the UK runtime imports an edited file, so no NZ, US, BE or transport key moves. - The identity tool's receipts: - every receipt gains geographic_clones; - E4 gains rewritten_later, rewritten_later_still_equal, held_to_tolerance and lha_category_scope, and E5 gains rewritten_later; - E8's band donor block takes the forms above. - On an invented frame run through the real stages, the parent's E4 fails at one clone on would_claim_uc, brma and is_apprentice, and at three clones on would_claim_uc and brma. This one passes at both. On an invented frame through was_wealth's allocation and student_loans, the parent's E5 fails on student_loan_balance, and this one reports it under rewritten_later. Tests: - Engine-free, the driver: - K = 1 and 2 through main: the node parameter, the transform and the engine at K, one engine identity, the sidecar's engine provenance at both and its batch record at K = 2 only, K rows per source household, and the sidecar's count; - --n-clones outside 1 to 100 is refused; - --release-candidate with two clones is refused by the parser and by the preparation; - a fixture of another clone count is refused; - a profile directory that is a file, holds another schema's profile or lies inside the repository is refused before any node runs; - the Logbook identity takes K only above one clone; - the spine run plain and profiled gives the same node keys and one record per node, and a profiled run over the plain store hits every node and writes no record. - Also engine-free: - the profile proxy is no kernel class and hashes what its kernel hashes; - the national problem is the first national node to read the frame, and it refuses a clone-first spine at one, two and three clones in one sentence; - the spine engine takes K, and its provenance does not; - the private-school parameters are the ones the engine-population declaration restates; - every engine stage after geographic_expand is an open surface, and the three temporary engine frames keep the clone index: an engine built for two clones records two simulations a call, and one without the index. - The identity tool: - the clone-count checks and their refusals; - the later-writer map against the manifest, and the rewritten columns the store still holds; - E5 on the student_loans re-split, and through main on an artifact without the regional home values' rank basis; - the scope's reversals against the carriers the artifact keeps; - E4 on a frame the stages built, at one and three clones, through main; - brma and is_apprentice on the final frame's flags and weights; - the CGT layers' fold, and the apprentice walk's hold and its tolerance; - copies_recomputed - copies in the exact-multiple family test; - E8 in the stage's capped forms; its funding rule on the stage's output at the floor, on a correction-sized move and on a stratum funded past its share; and spi_band_donors' count-free refusal texts. - Engine tier, new: packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py runs the H2 spine at one clone and at four, on invented data only. It checks that: - every stage runs, and every engine call from geographic_expand on runs four simulations, in as many calls as at one clone; - private_school_attendance takes the pooled restatement at four; - the clone-index invariant holds at every node, and each node keeps one clone's household mass; - clone 0 is the one-clone frame through frs_brma, except for the pooled draws and the BRMA of the households whose pooled draws differ, whose LHA_category the engine computes from them; - spi_support_channel copies, and spi_income_band_donors seats, the one-clone households in clone 0; - spi_income_band_donors funds every clone's donors at one mass scale, the smallest any clone's region allows, so each clone-0 donor weighs one clone's times the ratio of the two scales, and in each funding stratum of clone 0 every incumbent takes one ratio, one where a stratum has no donors; where the scales are equal every ratio is one. On H2's capped rung the scale at four clones is below one clone's, so K x clone-0 weights do not nest there; whether the stage takes a scale per clone is open; - every later output is on every clone, within a weighted-share tolerance; - the clone-count and fold receipts appear at the stages and clone counts they should. 13 pass. Run with one thread per native library, as CI runs it, the module took 2.6 times as long as the H2 parity test beside it. CI (.github/workflows/test.yml): the engine-uk job's cap goes from 60 to 120 minutes, about 1.6 times its expected slowest leg. Its legs took 31 to 46 minutes on main on 2026-11-11, and the new module adds about 27 minutes at the parity test's 10 minutes in that job. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 9 +- .gitignore | 5 + .../931-band-donors-per-clone.changed.md | 2 +- changelog.d/931-uk-clone-receipts.changed.md | 2 +- ...31-uk-engine-blocks-pool-shares.changed.md | 2 +- ...31-uk-fits-fold-geographic-clones.added.md | 2 +- changelog.d/931-uk-spine-clone-count.added.md | 19 + .../931-uk-spine-geographic-expand.changed.md | 4 +- .../build/uk_runtime/graph_national.py | 29 +- .../build/uk_runtime/spi_band_donors.py | 31 +- .../microcosm/build/uk_runtime/spine_build.py | 243 ++++- .../test_uk_engine_population_declaration.py | 256 +++++ .../tests/engine_free/uk/test_uk_frs_spine.py | 344 ++++++ .../tests/engine_free/uk/test_uk_graph.py | 46 +- .../uk/test_uk_identity_stability_receipts.py | 979 ++++++++++++++++++ .../engine_free/uk/test_uk_national_graph.py | 62 ++ .../uk/test_uk_private_school_attendance.py | 17 + .../uk/test_uk_rules_engine_provenance.py | 34 + .../engine_free/uk/test_uk_spi_band_donors.py | 219 +++- .../uk/test_acceptance_h_clone_first.py | 956 +++++++++++++++++ tools/verify_uk_identity_stability.py | 925 +++++++++++++++-- 21 files changed, 4044 insertions(+), 142 deletions(-) create mode 100644 changelog.d/931-uk-spine-clone-count.added.md create mode 100644 packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5696647f06..460ce9385d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,8 +32,11 @@ defaults: # frees its runner from the org's shared 60-runner pool. Each cap follows the # job's slowest successful leg on 2026-11-09, rounded up to a multiple of 5 # minutes: 1.5 times it for engine-free (109 minutes) and engine-us (73), twice -# for engine-uk (45) and integration-uk (8), and for each shorter job at least -# 5 minutes and at least twice its slowest run. +# for integration-uk (8), and for each shorter job at least 5 minutes and at +# least twice its slowest run. The engine-uk cap is about 1.6x its expected +# slowest leg with microcosm#931's clone-first H2 test: legs took 31-46 minutes +# on main on 2026-11-11, and that test's module runs about 2.6 times as long as +# the H2 parity test (10 minutes in this job), so about 27 minutes more. # Every job names its runner image. `ubuntu-latest` is whichever image GitHub # assigns: on 2026-11-10 it gave one engine-free job ubuntu-26.04 and another @@ -133,7 +136,7 @@ jobs: needs: select-countries if: needs['select-countries'].outputs.run_uk == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 120 strategy: fail-fast: false matrix: diff --git a/.gitignore b/.gitignore index 44ce9ecc74..0a0316a4d0 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,8 @@ tools/route_a/route_a.env # file holds licensed-data review values and is never committed PRIVATE_review.csv signoff-draft/ + +# Spine stage profiles (--stage-profile-dir, microcosm#931): a record keeps the +# message of a failure its node raised, so profiles stay private +stage_profile.json +.stage_profile.json.lock diff --git a/changelog.d/931-band-donors-per-clone.changed.md b/changelog.d/931-band-donors-per-clone.changed.md index a4ddff4034..dc444cf911 100644 --- a/changelog.d/931-band-donors-per-clone.changed.md +++ b/changelog.d/931-band-donors-per-clone.changed.md @@ -3,5 +3,5 @@ The SPI income band donors (`spi_income_band_donors`) are seated within each geo - **Funding.** A donor's funding stratum is its region within its clone, a derived stratum (the manifest's `funding_strata` stays `[region]`), so every clone's regions conserve their own mass. One mass scale, the smallest any clone's region allows, serves every clone, so each band keeps one donor weight and the stage gate's and the identity tool's band-mass reconciliations hold as before. The exact-total correction still lands on one incumbent row, never on a donor, so a band's donors stay equal bit for bit at any clone count. The seating nests exactly at any clone count. Clone 0's weights and funding factors nest, its weights as the one-clone weights over the clone count, only while the common scale is the one a one-clone spine takes: one at both, or bound by a region of clone 0. Where another clone's region binds, every clone's donors, clone 0's included, take the lower scale. Even while they nest, the weights nest to rounding, not bitwise: the corrected row, which can sit in any clone, moves by a few ulps of the total, and the 1/K split rounds when the clone count is not a power of two, so uncorrected rows also miss by about one ulp then. The mass record's reason is unchanged, so neither the frame's content identity nor the coverage manifest's restated reason moves. - **Receipt.** The evidence gains `clone_count`, and `funding_strata` lists `household_clone_index` where the column is present (the funding rows' labels then read `region|clone`). `expected_donor_households`, `expected_donor_count` and the donor counts cover every clone; `donor_weight` and `planned_donor_weight` are one clone's weights. Under the receipts' floor of three (`receipt_counts`), the eligible household counts, in all and per band, are counted in source households and rounded in place of the exact counts, each band's being clone 0's, which a one-clone spine counts alike; `seating_scale` reads `full` or `capped`, since an exact scale below one gives back the eligible count; a band's `carrier_mean_age` reads `<3` when its carriers come from fewer than three source households or the band seats fewer than three households per clone; and `carrier_region_counts` and the funding rows' `donor_households` are rounded. At full seating the band rows' donor and carrier counts stay exact row counts, which the gate reads; on a capped seating they are rounded source households, and the gate recomputes the seats from the frame. - **Gate.** The `spi_income_band_donor_support` check refuses a receipt without a positive integer `clone_count`. A donor's weight times the clone count must stay within the maximum, a band's donors must split evenly across the clones, and at full seating a band seats its plan and never fewer than the clone count times the minimum. The seating scale is read as `full` or `capped`, and a value in neither form is named by its type alone. At one clone the checks and their messages are as before, apart from the seating scale's form. -- **Refusals.** A clone short of candidates for a band is refused naming the band, without a count. A non-integer, missing or negative clone index, or an index from 0 to K - 1 that no household carries, is refused by count. +- **Refusals.** A clone short of candidates for a band is refused naming the band, without a count. A non-integer, missing or negative clone index, or an index from 0 to K - 1 that no household carries, is refused in a fixed sentence naming the column (and, for a column that is not integer, its dtype), with no count. - **What moves.** At K = 1, no value: on synthetic frames the seating, the carriers, every weight, the funding factors, the mass and seating scales, the mass record and the frame's content identity after the stage are bit for bit the parent commit's, with or without a one-clone index. The receipt takes the forms above. The stage's draw, weight-rule, funding, seating-scale and mass-scale texts and its notes change in `sources.yaml`, `source_stages.json` and the stage dictionary, so the stage's contract hash, the H2 fixture's declarations and the coverage manifest's 25 source digests are regenerated. `spi_income_band_donors` and every node after it re-key, as do the spine gates and the stages that bind `stage_health`. diff --git a/changelog.d/931-uk-clone-receipts.changed.md b/changelog.d/931-uk-clone-receipts.changed.md index c4caf5e99d..c3914ced0c 100644 --- a/changelog.d/931-uk-clone-receipts.changed.md +++ b/changelog.d/931-uk-clone-receipts.changed.md @@ -1,4 +1,4 @@ -The UK stage receipts and stage checks now read a spine of more than one geographic clone (#931; María's rulings of 10 Nov). Every new count is of distinct true source households, rounded through `receipt_counts`. The spine still builds one clone, so no value and no gate verdict moves. At one clone only the `spi_support_channel` and `spi_income_band_donors` receipts change, with the band donor check's details and failure texts and the clone count's refusal texts. +The UK stage receipts and stage checks now read a spine of more than one geographic clone (#931; María's rulings of 10 Nov). Every new count is of distinct true source households, rounded through `receipt_counts`. The spine still builds one clone by default, so no value and no gate verdict moves. At one clone only the `spi_support_channel` and `spi_income_band_donors` receipts change, with the band donor check's details and failure texts and the clone count's refusal texts. - **Fold receipts.** Every FRS-trained fit folds its training rows through `household_lineage.fold_geographic_clones`: the SPI income stage's reference and stage-2 forests, the SPI employment and housing shells, the UC reporter redraw, UC capital and salary sacrifice. On more than one clone, each of these stages' evidence gains `geographic_fold` (`household_lineage.geographic_fold_receipt`). It gives the clone count; the source households the fit selected, and those whose clones differ on what the fit reads, both rounded as every receipt count is; and the share that differ, given only where ten or more of the selected households differ and none or ten or more do not, and otherwise the count of those that differ again. A one-clone spine's evidence is unchanged. - **Twin check.** `cross_fit_transport` takes `twin_identities`, read only to find each recipient's twin among the training rows. The SPI income stage passes each copy's twin through its forests' fold, so the check reaches the copies of every clone. Each copy's uniforms stay keyed on its own identity; on one clone the two keys are the same. diff --git a/changelog.d/931-uk-engine-blocks-pool-shares.changed.md b/changelog.d/931-uk-engine-blocks-pool-shares.changed.md index d31cbd700a..2912b36646 100644 --- a/changelog.d/931-uk-engine-blocks-pool-shares.changed.md +++ b/changelog.d/931-uk-engine-blocks-pool-shares.changed.md @@ -3,4 +3,4 @@ Per-clone engine blocks now stand for the pool when the pool's clones differ (#9 - **What exact means.** `engine_population_representation.exact` now holds only when every block applied its correction, passed its checks and got a clean verdict, and then means equal to one engine over the pool to float32 rounding, not bitwise. `exact_basis` (`pool_share_injection_with_population_guard`) names this basis, and `exact_scope`, which the release's `engine_measurement` limitation copies, is rewritten. A receipt without `exact_basis` predates the change, when exact meant identical copies. The measure step records a violation, which leaves the run not releasable. The release verdict, the local pre-flight, the dense assembler and the data contract read the same fields as before; none requires `exact_basis`. - **Spine batches.** A spine engine call whose frame carries `household_clone_index` with more than one clone runs one simulation per clone, each reading the pool's shares (`c_b = S_b/S`) and judged under a pass that raises. A frame without the column, or with one clone, takes one simulation as before. Stage 52 (`private_school_attendance`) restates policyengine-uk's attendance over the whole frame from household-local engine values, with microdf's `MicroSeries.percentile_rank` and the engine's own parameters. Over one simulation it stores the engine's flags after checking the restatement bit for bit; clone by clone it stores the restatement. Its evidence gains `rank_route` and `engine_parity_checked`. - **The shared pieces.** The country-agnostic machinery (pool shares, the checks, injection, the formula-run recorder, judging, `EngineBlock` and batches) is `microcosm.frame.adapters.engine_population`, and the UK declaration is `microcosm.frame.adapters.policyengine_uk_population`. Each block and batch first checks the declaration against the installed policyengine-uk; a single block and an unbatched spine call never read it. A re-pin that changes the whole-population list or a pinned formula source therefore leaves every dense block uncorrected and the run not releasable, and refuses every batched spine call, until the declaration is reviewed again. -- **What moves.** Every spine is one clone (`geographic_expand` writes `household_clone_index`, 0 throughout), so every spine call takes one simulation and no spine data moves (frame, H5, stored flags), but stage 52's evidence gains its two fields on both lines, so its stage-evidence digest moves. Identical copies take `c_b` within float64 rounding of 1, so the correction leaves their measures bitwise as they were, and a single block, the national line's, gains none of the new receipt fields. Today's dense receipts gain `exact_basis`, `share_correction`, `population_reach` and `annual_income_projection_factors`; `block_sensitivity` now names engine variables, so `present_in_this_run` can list them, drops the SLC entry and gains `corrected_formulas`; and each block's resolver receipt gains `engine_pool_shares` and `engine_population_reach`. Every spine stage and the full and national target kernels re-key, and the engine-calling stages also bind the two new modules. The annual-income factor, the one data change, has its own fragment. +- **What moves.** A spine built at the default `--n-clones` is one clone (`geographic_expand` writes `household_clone_index`, 0 throughout), so every spine call takes one simulation and no spine data moves (frame, H5, stored flags), but stage 52's evidence gains its two fields on both lines, so its stage-evidence digest moves. Identical copies take `c_b` within float64 rounding of 1, so the correction leaves their measures bitwise as they were, and a single block, the national line's, gains none of the new receipt fields. Today's dense receipts gain `exact_basis`, `share_correction`, `population_reach` and `annual_income_projection_factors`; `block_sensitivity` now names engine variables, so `present_in_this_run` can list them, drops the SLC entry and gains `corrected_formulas`; and each block's resolver receipt gains `engine_pool_shares` and `engine_population_reach`. Every spine stage and the full and national target kernels re-key, and the engine-calling stages also bind the two new modules. The annual-income factor, the one data change, has its own fragment. diff --git a/changelog.d/931-uk-fits-fold-geographic-clones.added.md b/changelog.d/931-uk-fits-fold-geographic-clones.added.md index 64149fafc4..a592a3e636 100644 --- a/changelog.d/931-uk-fits-fold-geographic-clones.added.md +++ b/changelog.d/931-uk-fits-fold-geographic-clones.added.md @@ -1,7 +1,7 @@ The UK fits trained on the frame's FRS rows now fold geographic clones into each survey record before they fit (#931). This covers the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. - **How the fold works.** Each fit passes what it reads: its predictors and targets, or the columns they derive from. A record's clones that agree on it keep one training row, the lowest-id one among them (clone 0 whenever clone 0 is selected and agrees, whatever the row order), with the weight of those clones summed exactly. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own, each variant with its own clones' summed weight. SPI, high-value and CGT copies stay rows of their own, as now. -- **No change today.** Today's spine is one geographic clone (`geographic_expand` runs at K = 1), so every fit's rows and weights stay as they were, value for value. +- **No change today.** The default spine is one geographic clone (`geographic_expand` runs at K = 1), so every fit's rows and weights stay as they were, value for value. - **What changes once the spine clones before enrichment:** - the fits see each record once wherever its clones agree, and every clone's value at its share wherever they differ; - the reference transport meets no duplicate twins from agreeing clones; diff --git a/changelog.d/931-uk-spine-clone-count.added.md b/changelog.d/931-uk-spine-clone-count.added.md new file mode 100644 index 0000000000..fba9cc35e9 --- /dev/null +++ b/changelog.d/931-uk-spine-clone-count.added.md @@ -0,0 +1,19 @@ +The UK spine build takes its geographic clone count (#931). `tools/build_uk_frs_spine.py --n-clones K` builds the clone-first spine at K clones, K a whole number from 1 to 100, the `geographic_support` copy-id field's bound. Without the flag the spine builds one clone, as before. K goes to the `geographic_expand` node, to the transform that makes the clones and to the rules engine. Every engine call after `geographic_expand` runs one simulation per clone, and the engine refuses a frame that carries another count. The engine identity does not take K, so it is the same at every K. + +- **Guards.** A release candidate takes one clone until #931 group (c) pins the clone count a release builds at. `--release-candidate` with any other K is refused when the arguments are parsed, and again when the full build marks a parsed spine request a release candidate. A `--synthetic-fixture-dir` whose `config.geographic_clones` differs from `--n-clones` is refused while the spine is prepared, before any stage runs. +- **Sidecar and Logbook.** The build sidecar records `geographic_clone_count`, read from the frame's `household_clone_index`; a frame without that column counts as one clone. Its rules engine record keeps the engine's provenance at every K and adds the adapter's batch record only when the final frame carries more than one clone, as before. A spine of more than one clone binds its count into the Logbook identity digest; a one-clone build's digest is unchanged. +- **Memory guard.** `--stage-profile-dir` writes, for each spine graph node and under its id, the node's entry, peak and exit RSS and its wall time to `stage_profile.json`. The profiled kernels key as the kernels they wrap, so node keys do not move and a profiled run reuses an unprofiled run's store. A record keeps the message of a failure its node raised, so the directory must stay private. Before any node runs, the build refuses a directory inside the repository, a path that is not a directory, and a `stage_profile.json` there that is not a stage profile of the current schema, which the profiler would refuse only after the first node had run. `.gitignore` ignores `stage_profile.json` and its lock file. +- **National line.** Every spine from #931 group (b) on carries `household_clone_index`, at one clone too, and the national line drops the clone indices from its export only from group (c) on. So the national problem, the first national node that reads the bound frame, refuses such a spine before it resolves anything, in one fixed sentence (María's ruling of 10 Nov, decision 4). Without it, a national build on a one-clone spine would solve in full and then fail its terminal degenerate-surface gate. +- **Identity tool** (`tools/verify_uk_identity_stability.py`). + - `--spine-sidecar` takes the spine's build sidecar. The tool refuses a frame whose clone count differs from the sidecar's `geographic_clone_count` or, where the spine ran `cgt_support_split`, from the count that stage counted its copies on; the refusal gives only the clone counts. It refuses in a fixed sentence a sidecar that lacks those records, or whose output digest is not `--input-h5`'s. Every receipt records the counts under `geographic_clones`. + - E4's rules engine takes the frame's clone count (`PolicyEngineUKEngine(clones=K)`), so its `LHA_category` call runs clone by clone, as the stages' calls did. + - E4 and E5 no longer compare with the store a column that a later stage declares writing in the packaged source-stage manifest. They name it under `rewritten_later`, with those stages (María's ruling of 10 Nov). A rewrite the receipt's scope takes back does not count: `salary_sacrifice` and `regional_home_values`, where the artifact keeps the column each reversal reads. E5 reports `student_loan_balance`, which `student_loans` re-splits. E4 reports four flags that later take-up and coherence stages rewrite: `would_claim_uc`, `would_claim_pc`, `would_claim_child_benefit` and `child_benefit_opts_out`. It also reports the six childcare columns that `childcare_copy_take_up` declares rewriting, which by its notes it re-draws on copies only. So E4 also names, under `rewritten_later_still_equal`, the rewritten columns the store still holds bitwise. + - E4 recomputes on what its stages read. `LHA_category` is materialized on the frame `frs_brma` read, as far as the artifact restores it. Columns first written after that stage are dropped, and the flags `frs_take_up` drew that later stages rewrite are put back from the recompute. `lha_category_scope` names what stays as stored, with the stages that write it, `salary_sacrifice` among them, since its reversal takes back only the records it converted. The household weights are taken back through the band donors' funding and the SPI support channel's declared prior-mass shares, to rounding. `is_apprentice` is held exactly or, where those weights move the walk by rounding, to the walk on them, and `held_to_tolerance` then names it. + - E8's band donor block gives the layer in the forms of the stage's capped receipt, at every seating: + - the donors and each band's donors, in rounded distinct source households; + - the stage's funding rows, with their masses and factor withheld; + - `funding_within_share`, the stage's funding rule recomputed from the stored weights, which E8 also checks (`band_donor_funding_share`). It holds each rebuilt factor at E8's weight tolerance (relative 1e-9), not the stage gate's few ulps, since the stage's exact-total correction on one incumbent row moves a small stratum's rebuilt factor by more than that. + + It no longer writes the funding factors, band weights, weighted taxpayers or mass scale. Its checks still compare them. +- **SPI income band donors.** The stage's clone-index refusals are fixed sentences naming the column, with no count. +- **What moves.** At one clone, no spine value and no gate verdict moves. The sidecar gains `geographic_clone_count`, and the national line refuses a spine from group (b) on at its first node that reads the frame. Node keys move from `spi_income_band_donors` on, and across the national line. The identity tool's receipts gain the fields above. diff --git a/changelog.d/931-uk-spine-geographic-expand.changed.md b/changelog.d/931-uk-spine-geographic-expand.changed.md index 57a8e50499..948bc97302 100644 --- a/changelog.d/931-uk-spine-geographic-expand.changed.md +++ b/changelog.d/931-uk-spine-geographic-expand.changed.md @@ -1,4 +1,4 @@ -The UK spine gets its geographic clones after its source stages (#931). `frs_hmrc_spine_leaves` and `frs_annual_income_spells`, which join the raw FRS tabs on raw person ids, move ahead of `frs_take_up`, and two new stages follow them: `geographic_expand` (G1) gives every FRS household K geographic clones, and `geographic_identity` (G1b) keys each clone. `frs_take_up`, `frs_person_draws`, `frs_household_draws` and `frs_brma` then run on every clone (María's ruling of 10 Nov, decision 3). The spine build runs one clone, so no value moves. Positional stage numbers in the #931 fragments, comments and tests still follow the order before this change: 14 is `frs_hmrc_spine_leaves`, 15 `frs_annual_income_spells`, 10 to 13 the draws and `frs_brma`, and 16 onward as before. +The UK spine gets its geographic clones after its source stages (#931). `frs_hmrc_spine_leaves` and `frs_annual_income_spells`, which join the raw FRS tabs on raw person ids, move ahead of `frs_take_up`, and two new stages follow them: `geographic_expand` (G1) gives every FRS household K geographic clones, and `geographic_identity` (G1b) keys each clone. `frs_take_up`, `frs_person_draws`, `frs_household_draws` and `frs_brma` then run on every clone (María's ruling of 10 Nov, decision 3). The spine build runs one clone by default, so no value moves. Positional stage numbers in the #931 fragments, comments and tests still follow the order before this change: 14 is `frs_hmrc_spine_leaves`, 15 `frs_annual_income_spells`, 10 to 13 the draws and `frs_brma`, and 16 onward as before. - **Order.** The two moved stages read only employment and self-employment income, age and gender. The draws read none of their outputs and write none of their inputs, and none of these stages sets weights. `frs_brma` stays the assembled checkpoint's last stage, so the assembled phase now covers the moved stages, G1, G1b and the draws, and the assembled gate admits `spi_support_channel`. - **Declarations.** The sources schema and `ALLOWED_SOURCE_OPERATION_KINDS` gain two strict operation kinds. @@ -8,7 +8,7 @@ The UK spine gets its geographic clones after its source stages (#931). `frs_hmr Each transform refuses any other declaration of its operation. G1 refuses any artifact, and G1b any but that one. - **Clone count.** No declaration names K. `uk_spine_graph(geographic_clones=K)` puts it on G1's node, a whole number from 1 to 100, and refuses K above 1 for a manifest without G1. So K enters that node's key and every later one, never a stage contract. - G1 is an EXPAND that conserves mass at design weights. Its kernel refuses a transform whose clone count differs from its node's, printing only the two counts. - - The spine build runs G1 at K = 1 and builds its engine with `clones=1`. `_rules_engine`'s `clones` defaults to None, so the full build's engine installs are unchanged. + - The spine build runs G1 at the clone count `--n-clones` asks for, one by default, and builds its engine with that count. `_rules_engine`'s `clones` defaults to None, so the full build's engine installs are unchanged. - The H2 registry builds G1 and its engine from the fixture's `config.geographic_clones`. - The dense line's `uk.full.identity` declares the geography key as a rewrite where the spine carries it, so the dense graph still compiles. - **Source columns.** `source_household_id`, `source_year` and `source_household_key` are G1's outputs. `spi_support_channel` no longer declares or writes them, and its copies inherit them. It and the dense expansion default them from a household's own ids only on a table without `household_clone_index`, and refuse, naming the columns, a clone-indexed table that lacks any of them. `source_person_id` and `source_benunit_id` are not minted (decision 5). Their names live in the `geographic_clones` leaf beside the clone-index names (`UK_GEOGRAPHIC_SOURCE_COLUMNS`, `SOURCE_YEAR_COLUMN`, `SOURCE_HOUSEHOLD_KEY_COLUMN`), and so does the geography key's (`GEOGRAPHY_HOUSEHOLD_KEY_COLUMN`, which `atomic_area_support.IDENTITY_COLUMN` re-exports). So G1, G1b, the SPI support channel and the dense line read one definition, which their keys hash. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index 5f05ed38ac..56f97f080b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -103,6 +103,7 @@ from .diagnostics import uk_target_geography_levels from .full_targets import CHRONICLE_SOURCE_CODEC, load_uk_national_target_inputs from .gate_scopes import uk_scoped_gate_manifest +from .geographic_clones import ladder_clone_index_column from .graph_build import SPINE_PROVENANCE_TYPE from .graph_calibration import ( UKCalibrationNodes, @@ -676,13 +677,38 @@ def materialize_uk_national_rows( return prepared, restore, rows, dict(evidence) +def refuse_clone_first_national_spine(frame: Frame) -> None: + """Refuse a bound spine whose households carry a geographic clone index. + + The clone-first spine's geographic_expand stage (microcosm#931) writes + ``household_clone_index`` on every spine from group (b) on, at one clone + too, and the national line drops the clone indices from its export only + from group (c) on. Before that, a national build on a one-clone spine + would solve in full and then fail its terminal degenerate-surface gate on + the constant indices. So the national problem, the first national node + that reads the frame, refuses such a spine before it resolves anything. + The message names the column and the group, never a count or a value. + """ + + column = ladder_clone_index_column("household") + if column in frame.table("household").columns: + raise ValueError( + f"The national line refuses a bound spine that carries {column!r}, " + "as every spine since microcosm#931 group (b) does at any clone " + "count: it builds on none until group (c) drops the clone indices " + "from its export." + ) + + class UKNationalProblemKernel(_NationalKernel): """One ordered national problem with the doctrine bound into it. ``resolver_factory`` is instance state like the gate kernels' coverage engine: the engine identity the run stands on is provenance recorded by the driver, and the dependency declaration on the capabilities binds the - installed engine version into the node key. + installed engine version into the node key. The first national node that + reads the bound frame, it refuses a clone-first spine before anything else + (:func:`refuse_clone_first_national_spine`, microcosm#931). """ ref = "uk.full.national_problem@1" @@ -755,6 +781,7 @@ def implementation_hash(self) -> str: def run(self, context: KernelContext) -> KernelResult: frame = context_frame(context) + refuse_clone_first_national_spine(frame) registry_document = _registry_artifact(context) with tempfile.TemporaryDirectory( prefix="microcosm-uk-national-measures-" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py index cf705aff25..4d1cd6dac7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_band_donors.py @@ -1140,8 +1140,9 @@ def _geographic_clone_indices(household: pd.DataFrame) -> np.ndarray | None: every household the index of its geographic clone in ``household_clone_index``, and its copies keep their root's; a household table without the column is one clone. Refuses a non-integer column, - a household without an index and a negative index, each by count or - dtype, never by id or value. + a household without an index and a negative index, each in a fixed + sentence naming the column (and, for a column that is not integer, its + dtype): never a count, an id or a value. """ column = SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN @@ -1155,18 +1156,16 @@ def _geographic_clone_indices(household: pd.DataFrame) -> np.ndarray | None: "SPI income band donors seat each geographic clone on an integer " f"{column!r} column; it has dtype {values.dtype}." ) - unindexed = int(values.isna().sum()) - if unindexed: + if bool(values.isna().any()): raise ValueError( - f"{unindexed} household row(s) carry no {column!r}; SPI income band " - "donors seat each geographic clone." + f"A household row carries no {column!r}; SPI income band donors seat " + "each geographic clone." ) indices = values.to_numpy(dtype=np.int64) - negative = int((indices < 0).sum()) - if negative: + if bool((indices < 0).any()): raise ValueError( - f"{negative} household row(s) carry a negative {column!r}; SPI income " - "band donors seat each geographic clone." + f"A household row carries a negative {column!r}; SPI income band " + "donors seat each geographic clone." ) return indices @@ -1176,18 +1175,18 @@ def _geographic_clone_count(indices: np.ndarray | None) -> int: Every clone from 0 to ``K - 1`` must carry a household: each one seats the band's seats, so a missing clone would leave its share of the - published mass unseated. The refusal counts the missing indices. + published mass unseated. The refusal is a fixed sentence naming the + column, with no count. """ if indices is None or not indices.size: return 1 clone_count = int(indices.max()) + 1 - absent = clone_count - int(np.unique(indices).size) - if absent: + if int(np.unique(indices).size) != clone_count: raise ValueError( - f"{absent} of the {clone_count} geographic clone indices from 0 to the " - "largest are carried by no household; SPI income band donors seat " - "clones 0 to K - 1." + f"A {SPI_GEOGRAPHIC_CLONE_INDEX_COLUMN!r} value from 0 to the largest is " + "carried by no household; SPI income band donors seat clones 0 to " + "K - 1." ) return clone_count diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index 7b9301e943..f7edb2c8d4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -10,7 +10,7 @@ import sys import time import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime from importlib import metadata @@ -50,6 +50,12 @@ ) from microcosm.build.plan import StageRecord from microcosm.build.run_outcome import classify_failure +from microcosm.build.stage_profile import ( + STAGE_PROFILE_FILENAME, + STAGE_PROFILE_SCHEMA_VERSION, + _load_profile_payload, + profile_stage, +) from microcosm.build.staging_cli import ( add_staging_arguments, validate_staging_arguments, @@ -125,7 +131,9 @@ uk_frs_spine_seed_frame, ) from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.geographic_expand import ( + GEOGRAPHIC_CLONE_LIMIT, UKGeographicExpandStageTransform, ) from microcosm.build.uk_runtime.geographic_identity import ( @@ -238,6 +246,10 @@ from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine from microcosm.graph import ( ContentStore, + Kernel, + KernelContext, + KernelRegistry, + KernelResult, compile_graph, run_graph, save_run_evidence, @@ -255,11 +267,19 @@ ) _RUNG_NAMED_EDGE_SIGNATURE = "The least populated classes in y have only 1 member" _RUNG_ABORT_EXIT_CODE = 3 -#: The spine's geographic clone count K (microcosm#931): one, until the spine -#: build takes a count. The graph's geographic_expand node, the transform it -#: runs and the engine all take it, and the node refuses a transform that -#: makes another number of clones. +#: The spine's geographic clone count K by default (microcosm#931): one, unless +#: ``--n-clones`` asks for another, from 1 to GEOGRAPHIC_CLONE_LIMIT. The +#: graph's geographic_expand node, the transform it runs and the engine all +#: take the count, and the node refuses a transform that makes another number +#: of clones. _GEOGRAPHIC_CLONES = 1 +#: Why a release candidate takes one clone (microcosm#931): the clone count a +#: release builds at is pinned only from group (c) on. +_RELEASE_CANDIDATE_CLONES_REFUSAL = ( + "--release-candidate builds the spine at one geographic clone " + "(--n-clones 1): the clone count a release builds at is pinned only from " + "microcosm#931 group (c) on." +) #: Declared outputs whose nonzero share no stage record reports: the root's #: area-conditioning carriers (frs_spine.FRS_AREA_CONDITIONING_COLUMNS, #: microcosm#931), which stay out of every receipt and of the sidecar's @@ -304,6 +324,22 @@ def _rung_sample_fraction(value: str) -> float: return fraction +def _clone_count_argument(value: str) -> int: + """``--n-clones``: a whole number from 1 to the copy-id field's bound.""" + + refusal = ( + "--n-clones must be a whole number from 1 to " + f"{GEOGRAPHIC_CLONE_LIMIT}, the geographic_support copy-id field's bound." + ) + try: + clones = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError(refusal) from error + if not 1 <= clones <= GEOGRAPHIC_CLONE_LIMIT: + raise argparse.ArgumentTypeError(refusal) + return clones + + def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( @@ -381,6 +417,30 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=UK_SAMPLE_SEED_DEFAULT, help=f"Raw FRS spine sampling seed (default: {UK_SAMPLE_SEED_DEFAULT}).", ) + parser.add_argument( + "--n-clones", + type=_clone_count_argument, + default=_GEOGRAPHIC_CLONES, + help=( + "Geographic clones of every FRS household, from 1 to " + f"{GEOGRAPHIC_CLONE_LIMIT} (default: {_GEOGRAPHIC_CLONES}; " + "microcosm#931). The geographic_expand stage makes them after the " + "stages that read the raw FRS tables, each at an equal share of its " + "household's weight, and every later engine call runs clone by " + "clone. A release candidate takes one." + ), + ) + parser.add_argument( + "--stage-profile-dir", + type=Path, + help=( + f"Optional directory for {STAGE_PROFILE_FILENAME}: each graph " + "node's entry, peak and exit RSS and its wall time, the spine's " + "memory guard (microcosm#931). A node's record keeps the message of " + "a failure it raised, so keep the directory private, outside every " + "staged output; the build refuses one inside the repository." + ), + ) parser.add_argument( "--was-tab", type=Path, @@ -466,6 +526,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("sample seed must be a non-negative integer.") if args.smoke and args.release_candidate: parser.error("non-release smoke builds refuse --release-candidate.") + if args.release_candidate and args.n_clones != 1: + parser.error(_RELEASE_CANDIDATE_CLONES_REFUSAL) if _is_sampled(args) and args.checkpoint_dir is not None: parser.error( "sampled spine builds refuse --checkpoint-dir; sampled artifacts " @@ -522,6 +584,10 @@ def _sample_token(args: argparse.Namespace) -> str: def _validate_args(args: argparse.Namespace) -> None: + # The parser refuses the pair too; the full build sets release_candidate + # on a spine request it has already parsed, so the refusal holds here. + if args.release_candidate and args.n_clones != 1: + raise ValueError(_RELEASE_CANDIDATE_CLONES_REFUSAL) if args.synthetic_fixture_dir is not None: if not args.synthetic_fixture_dir.is_dir(): raise ValueError( @@ -555,6 +621,9 @@ def _validate_args(args: argparse.Namespace) -> None: } if args.emit_nonzero_shares is not None: paths["emit_nonzero_shares"] = args.emit_nonzero_shares + if args.stage_profile_dir is not None: + _validate_stage_profile_dir(args.stage_profile_dir) + paths["stage_profile"] = args.stage_profile_dir / STAGE_PROFILE_FILENAME resolved: dict[Path, str] = {} for label, path in paths.items(): target = Path(path).expanduser().resolve() @@ -564,6 +633,68 @@ def _validate_args(args: argparse.Namespace) -> None: resolved[target] = label +def _logbook_run_config( + args: argparse.Namespace, + *, + stage_names: Sequence[str], + artifact_pins_digest: str, + synthetic_fixture: Mapping[str, object] | None, +) -> dict[str, object]: + """What the Logbook's identity digest is hashed from. + + A spine of more than one geographic clone is another build, so its clone + count joins it (microcosm#931); a one-clone build's digest stays as it + was. + """ + + run_config: dict[str, object] = { + "pipeline": _PIPELINE, + "stages": list(stage_names), + "artifact_pins_digest": artifact_pins_digest, + "spine_h5": str(args.spine_h5), + "synthetic_fixture": synthetic_fixture, + } + if args.n_clones != 1: + run_config["geographic_clones"] = args.n_clones + return run_config + + +def _validate_stage_profile_dir(directory: Path) -> None: + """Refuse a ``--stage-profile-dir`` before any node runs (microcosm#931). + + The profiler merges a node's record into ``stage_profile.json`` only once + the node has run, so a path that is not a directory, or a profile there + of another schema, would fail the build after its first node and lose + that node's result. A record keeps the message of a failure its node + raised, so the directory must also lie outside the repository. Each + refusal is a fixed sentence. + """ + + target = Path(directory).expanduser().resolve() + if target.exists() and not target.is_dir(): + raise ValueError("--stage-profile-dir must name a directory.") + repository = _REPOSITORY.resolve() + if (repository / "pyproject.toml").is_file() and ( + target == repository or repository in target.parents + ): + raise ValueError( + "--stage-profile-dir must lie outside the repository: a stage " + "profile keeps the message of a failure its node raised, so it " + "stays private." + ) + existing = target / STAGE_PROFILE_FILENAME + if existing.exists(): + try: + _load_profile_payload(existing) + except (OSError, ValueError): + raise ValueError( + f"--stage-profile-dir holds a {STAGE_PROFILE_FILENAME} that is " + "not a stage profile of schema version " + f"{STAGE_PROFILE_SCHEMA_VERSION}, so no node's record could be " + "added to it." + ) from None + + def _synthetic_fixture_evidence(source: Path | None) -> dict[str, object] | None: """Bind a non-release integration run to every file in its fixture.""" @@ -763,8 +894,8 @@ def _entity_row_counts(frame) -> dict[str, int]: def _rules_engine(*, clones: int | None = None) -> PolicyEngineUKEngine: """The pinned engine; ``clones``, the spine's clone count where the caller - knows it, makes the engine refuse a frame of any other clone count, and - ``None`` refuses none (microcosm#931).""" + knows it, makes the engine refuse a frame carrying another clone count, + and ``None`` refuses none (microcosm#931).""" try: import policyengine_uk # noqa: F401 @@ -993,6 +1124,14 @@ def _build_sidecar( "time_period": str(frame.metadata["time_period"]), "household_weight_kind": uk_household_weight_kind(frame).value, "household_weight_total": float(household_weight.values.sum()), + # The spine's geographic clone count K, read from the frame it + # describes: the --n-clones the build ran at, since the + # geographic_expand node refuses a transform that makes another + # count, and one where the frame carries no clone index + # (microcosm#931). + "geographic_clone_count": geographic_clone_count( + frame.table("household"), stage="The spine sidecar" + ), "entity_row_counts": _entity_row_counts(frame), "artifact_pins": artifact_pins, "resource_pins": resource_pins, @@ -1324,6 +1463,46 @@ def __getattr__(self, name: str): return getattr(transform, name) +class _ProfiledKernel(Kernel): + """A spine kernel that computes each node inside + :func:`~microcosm.build.stage_profile.profile_stage`, under the node's id, + in ``--stage-profile-dir`` (the memory guard ruled on 6 Nov, microcosm#931). + + It is the kernel it wraps to everything but ``run``: ``__getattr__`` + forwards every other attribute, the kernel's reference and capabilities + among them, and ``implementation_hash`` is the wrapped kernel's own bound + method, so each node keys, hits and stores as it does unprofiled. It + declares the graph's kernel protocol for the registry's check but no + reference of its own, so it is no kernel class of the UK runtime and the + binding rule holds the kernel it wraps. A record keeps the message of a + failure the node raised, so the directory is private. + """ + + def __init__(self, kernel: Kernel, directory: Path) -> None: + self._kernel = kernel + self._directory = directory + self.implementation_hash = kernel.implementation_hash + + def run(self, context: KernelContext) -> KernelResult: + with profile_stage(context.node.id, self._directory): + return self._kernel.run(context) + + def __getattr__(self, name: str): + kernel = self.__dict__.get("_kernel") + if kernel is None: + raise AttributeError(name) + return getattr(kernel, name) + + +def _profiled_kernels(kernels: KernelRegistry, directory: Path) -> KernelRegistry: + """``kernels``, each wrapped in :class:`_ProfiledKernel` under its ref.""" + + profiled = KernelRegistry() + for ref in kernels.refs(): + profiled.register(_ProfiledKernel(kernels.get(ref), directory)) + return profiled + + def _run_plan_with_spine_sampling( plan, *, @@ -1579,11 +1758,13 @@ def prepare_uk_spine_execution( """Bind source paths and lazy transforms without fitting models or writing files. Everything ``main`` decides before the graph runs lives here: the stage - roster, the private-input checks, the stage implementations (licensed or - synthetic fixture), the sampling root, the optional observation wrap, the - declared graph sources, and the gate nodes bound to the rules engine. - ``observer`` is the staging run bundle's stage observer when that bundle is - enabled; ``main`` supplies it, and a caller without telemetry passes none. + roster, the private-input checks, the geographic clone count, the stage + implementations (licensed or synthetic fixture), the sampling root, the + optional observation wrap, the declared graph sources, the gate nodes bound + to the rules engine and, with ``--stage-profile-dir``, the profiled + kernels. ``observer`` is the staging run bundle's stage observer when that + bundle is enabled; ``main`` supplies it, and a caller without telemetry + passes none. """ _validate_args(args) @@ -1592,12 +1773,16 @@ def prepare_uk_spine_execution( if spec.sources is None: raise ValueError("UK country spec has no source stages.") stages_by_name = spec.sources.stage_map() + # The clone count K enters the graph on the geographic_expand node, the + # transform that makes the clones and the engine, which refuses a frame + # carrying another count; never the engine's identity (microcosm#931). + clones = args.n_clones graph = uk_spine_graph( spec, source_mode="split", sample_fraction=args.sample_fraction, sample_seed=args.sample_seed, - geographic_clones=_GEOGRAPHIC_CLONES, + geographic_clones=clones, ) stage_names = _uk_spine_stage_names(spec) if ( @@ -1676,7 +1861,7 @@ def prepare_uk_spine_execution( raise ValueError("--aps-tab is required when aps_occupation is scheduled.") stages = [stages_by_name[name] for name in stage_names] synthetic_fixture = _synthetic_fixture_evidence(args.synthetic_fixture_dir) - engine = _rules_engine(clones=_GEOGRAPHIC_CLONES) + engine = _rules_engine(clones=clones) stochastic_contract = load_uk_take_up_contract() frs_release = load_uk_frs_release() hmrc_spine_transform = _GraphSourceTransform( @@ -1862,7 +2047,7 @@ def prepare_uk_spine_execution( if "geographic_expand" in stage_names: implementations["geographic_expand"] = UKGeographicExpandStageTransform( stage=stages_by_name["geographic_expand"], - clones=_GEOGRAPHIC_CLONES, + clones=clones, ) if "geographic_identity" in stage_names: implementations["geographic_identity"] = UKGeographicIdentityStageTransform( @@ -2032,6 +2217,17 @@ def prepare_uk_spine_execution( f"fixture={fixture_stage_names!r}, current={tuple(stage_names)!r}." ) implementations = dict(fixture_implementations) + # The fixture's transforms and engine clone at its own + # config.geographic_clones (microcosm#931), so the request must ask for + # that count; the geographic_expand node would refuse another, only + # after the source stages had run. + fixture_expand = implementations.get("geographic_expand") + if fixture_expand is not None and fixture_expand.clones != clones: + raise ValueError( + "--n-clones differs from the synthetic fixture's " + "config.geographic_clones, the count its transforms and engine " + "clone at (microcosm#931)." + ) fixture_by_name = {stage.stage: stage for stage in fixture_stages} implementations["frs_hmrc_spine_leaves"] = UKFRSHMRCSpineLeavesStageTransform( _synthetic_fixture_input(args.synthetic_fixture_dir, "frs_raw"), @@ -2107,6 +2303,10 @@ def prepare_uk_spine_execution( register_spine_gate_kernel( kernels, spec=spec, engine=engine, engine_identity=engine_identity ) + if args.stage_profile_dir is not None: + # Each node's RSS and wall time, under its node id; the keys are the + # unprofiled kernels' (microcosm#931). + kernels = _profiled_kernels(kernels, args.stage_profile_dir.expanduser()) return PreparedUKSpineExecution( graph=graph, kernels=kernels, @@ -2213,13 +2413,12 @@ def _main(argv: list[str] | None = None) -> int: _role_pins({**artifact_pins, **input_artifact_pins}) ) synthetic_fixture = prepared.synthetic_fixture - run_config = { - "pipeline": _PIPELINE, - "stages": list(stage_names), - "artifact_pins_digest": state.input_pins_digest, - "spine_h5": str(args.spine_h5), - "synthetic_fixture": synthetic_fixture, - } + run_config = _logbook_run_config( + args, + stage_names=stage_names, + artifact_pins_digest=state.input_pins_digest, + synthetic_fixture=synthetic_fixture, + ) state.identity_digest = hashlib.sha256( canonical_json_bytes(run_config) ).hexdigest() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_engine_population_declaration.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_engine_population_declaration.py index 6bc9f8791f..dbffbf29da 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_engine_population_declaration.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_engine_population_declaration.py @@ -5,16 +5,56 @@ calls batch by, which must be the ladder frame's clone index, and the frame columns each share formula's allocation key adds, which the measure step sums for the pool-share correction. + +The spine side of that batching is held here too. The adapter runs a +call over a frame that carries the clone index clone by clone, and one over a +frame without it as one simulation over the rows it holds, so no engine +stage after geographic_expand may hand the engine a frame that lost the +index: each is an open surface, given the whole frame, and the three stages +that build a temporary frame for the engine keep the index on it. """ from __future__ import annotations +import numpy as np +import pandas as pd + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime import graph, graph_kernels +from microcosm.build.uk_runtime.frs_take_up import UKTakeUpPopulationPolicy from microcosm.build.uk_runtime.full_measure import UK_WEIGHT_SHARE_FORMULA_INPUTS +from microcosm.build.uk_runtime.geographic_expand import GEOGRAPHIC_EXPAND_STAGE_NAME +from microcosm.build.uk_runtime.national_frame import uk_national_frame +from microcosm.build.uk_runtime.pension_credit_take_up import ( + PENSION_CREDIT_MIXED_AGE_SAVING, + redraw_pension_credit_take_up, +) from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column +from microcosm.build.uk_runtime.spi_support import support_channel_column +from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract +from microcosm.build.uk_runtime.uc_reporter_redraw import redraw_spi_reported_uc +from microcosm.build.uk_runtime.uc_take_up import ( + UC_AWARD_BAND_LABELS, + UC_FAMILY_TYPES, + redraw_uc_take_up, +) +from microcosm.frame import WeightKind +from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine from microcosm.frame.adapters.policyengine_uk_population import ( UK_ENGINE_POPULATION, UK_SPINE_ENGINE_BATCH_COLUMN, ) +from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame +from test_support.microcosm_build.uk_uc_reporter_redraw import ( + _frame as _reporter_frame, +) +from test_support.microcosm_build.uk_uc_reporter_redraw import ( + _StubEngine as _ReporterEngine, +) +from test_support.microcosm_build.uk_uc_reporter_redraw import _StubQRFFactory + +#: The clone count the temporary frames are checked at. +_CLONES = 2 def test_spine_engine_calls_batch_by_the_ladder_households_clone_index(): @@ -28,3 +68,219 @@ def test_the_share_inputs_the_build_sums_are_the_declared_key_adds(): } # Tuples: the same columns in the same order. assert dict(UK_WEIGHT_SHARE_FORMULA_INPUTS) == declared + + +def test_every_engine_stage_after_the_clones_is_given_the_whole_frame(): + """Every stage after geographic_expand whose node binds the engine + modules (``graph_kernels._STAGE_HELPER_MODULES``; test_uk_graph holds + these to exactly the stages that call the engine or take one) declares no + consumes, an open surface, so its node's slices hold the household clone + index the adapter batches by.""" + + order = [stage.stage for stage in load_country_spec("uk").sources.stages] + after = order[order.index(GEOGRAPHIC_EXPAND_STAGE_NAME) + 1 :] + engine_modules = set(graph_kernels._ENGINE_MODULES) + engine_stages = [ + stage + for stage in after + if engine_modules <= set(graph_kernels._STAGE_HELPER_MODULES.get(stage, ())) + ] + # The premise: the engine stages after the clones include the three that + # build a temporary frame, checked below. + assert { + "uc_reporter_redraw", + "uc_take_up", + "pension_credit_take_up", + } <= set(engine_stages) + spine = graph.uk_spine_graph() + for stage in engine_stages: + assert graph._STAGE_CONSUMES[stage] is None, stage + household = { + column + for item in spine.node(stage).inputs + if item.entity == "household" + for column in item.columns + } + assert UK_SPINE_ENGINE_BATCH_COLUMN in household, stage + + +class _ClonedCallRecorder(PolicyEngineUKEngine): + """A UK engine built for a spine of :data:`_CLONES` clones whose + simulations are a stand-in's. Each call records how many simulations the + adapter runs over the frame it is handed, by the adapter's own decision + and refusals: one per clone where the frame carries the clone index, one + where it lost it, and a refusal where it carries another count. It then + answers with the stand-in's values.""" + + def __init__(self, stand_in) -> None: + super().__init__(clones=_CLONES) + self.stand_in = stand_in + self.batches: list[int] = [] + + def materialize(self, bundle, variables, period, **kwargs): + self.batches.append(self.simulation_batches(bundle)) + return self.stand_in.materialize(bundle, variables, period, **kwargs) + + +class _UCEngine: + """Every unit is paid an award; nobody is a child.""" + + country = "uk" + + def materialize(self, frame, variables, period, *, dataset_period=None): + units = len(frame.table("benunit")) + values = { + "universal_credit": np.full(units, 900.0), + "legacy_benefits_closed": np.zeros(units, dtype=bool), + "is_child_or_qualifying_young_person_for_universal_credit": np.zeros( + len(frame.table("person")), dtype=bool + ), + } + return {name: values[name] for name in variables} + + +class _PensionCreditEngine: + """Unit 1 is entitled to Guarantee Credit, unit 2 to Savings Credit only.""" + + country = "uk" + + def materialize(self, frame, variables, period, *, dataset_period=None): + ids = frame.table("benunit")["benunit_id"].to_numpy() % 1_000_000_000 + values = { + PENSION_CREDIT_MIXED_AGE_SAVING: ids == 2, + "guarantee_credit": np.where(ids == 1, 40.0, 0.0), + "savings_credit": np.where(ids == 2, 15.0, 0.0), + "is_pension_credit_eligible": ids != 3, + "pension_credit_assessable_capital": np.zeros(len(ids)), + "pension_credit_deemed_income": np.zeros(len(ids)), + } + return {name: values[name] for name in variables} + + +def _uc_take_up_frame(): + person = pd.DataFrame( + { + "person_id": [11, 21, 31], + "person_benunit_id": [1, 2, 3], + "person_household_id": [1, 2, 3], + "age": [30, 40, 50], + "is_uc_claimant": [True] * 3, + "is_benunit_head": [True] * 3, + "is_parent": [False] * 3, + "is_claimant_or_partner": [True] * 3, + "universal_credit_reported": [1_000.0, 0.0, 0.0], + **{ + column: [0.0] * 3 + for column in ( + "child_tax_credit_reported", + "working_tax_credit_reported", + "housing_benefit_reported", + "esa_income_reported", + "income_support_reported", + "jsa_income_reported", + ) + }, + } + ) + benunit = pd.DataFrame( + { + "benunit_id": [1, 2, 3], + "dependent_children": [0, 0, 0], + "would_claim_uc": [True, True, False], + } + ) + household = pd.DataFrame( + { + "household_id": [1, 2, 3], + "region": ["WALES", "WALES", "NORTHERN_IRELAND"], + "source_household_id": [1, 2, 3], + } + ) + return uk_national_frame( + person=person, + benunit=benunit, + household=household, + household_weights=np.asarray([10.0, 10.0, 10.0]), + weight_kind=WeightKind.IMPORTANCE, + time_period="2024", + ) + + +def _pension_credit_frame(): + return uk_national_frame( + person=pd.DataFrame( + { + "person_id": [11, 21, 31], + "person_benunit_id": [1, 2, 3], + "person_household_id": [1, 2, 3], + "age": [80, 78, 70], + "pension_credit_reported": [0.0, 0.0, 500.0], + } + ), + benunit=pd.DataFrame( + { + "benunit_id": [1, 2, 3], + "would_claim_pc": [True, True, False], + support_channel_column("benunit"): ["frs", "frs", "spi"], + } + ), + household=pd.DataFrame({"household_id": [1, 2, 3], "region": ["WALES"] * 3}), + household_weights=np.asarray([10.0, 10.0, 10.0]), + weight_kind=WeightKind.IMPORTANCE, + time_period="2024", + ) + + +def _uc_targets() -> dict[tuple[str, int], float]: + targets = { + (name, index): 0.0 + for name, _ in UC_FAMILY_TYPES + for index in range(len(UC_AWARD_BAND_LABELS)) + } + targets[("SINGLE", 0)] = 10.0 + return targets + + +def test_the_temporary_engine_frames_keep_the_clone_index(): + """The three stages that hand the engine a frame of their own making + (the reporter screen, the claim-everyone award and the Pension Credit + entitlement with its stored saving) keep the household clone index on + it, so an engine built for a spine of two clones runs each of their calls + clone by clone, as it does the calls over the stage's own frame.""" + + runs = { + "uc_reporter_redraw": lambda frame, engine: redraw_spi_reported_uc( + frame, engine=engine, qrf_factory=_StubQRFFactory() + ), + "uc_take_up": lambda frame, engine: redraw_uc_take_up( + frame, + engine=engine, + population_policy=UKTakeUpPopulationPolicy( + adult_age=18, state_pension_age=66, instant="2024-01-01", source="test" + ), + targets=_uc_targets(), + closure_rates={"all": 0.8}, + minimum_pool_sources=1, + ), + "pension_credit_take_up": lambda frame, engine: redraw_pension_credit_take_up( + frame, engine=engine, contract=load_uk_take_up_contract() + ), + } + frames = { + "uc_reporter_redraw": (_reporter_frame(), _ReporterEngine()), + "uc_take_up": (_uc_take_up_frame(), _UCEngine()), + "pension_credit_take_up": (_pension_credit_frame(), _PensionCreditEngine()), + } + # The temporary frame is each stage's only call but Pension Credit's, + # whose first call reads the stage's own frame for the saving it stores. + calls = {"uc_reporter_redraw": 1, "uc_take_up": 1, "pension_credit_take_up": 2} + for stage, run in runs.items(): + base, stand_in = frames[stage] + engine = _ClonedCallRecorder(stand_in) + run(clone_first_frame(base, _CLONES), engine) + assert engine.batches == [_CLONES] * calls[stage], stage + # The control: a frame without the index takes one simulation a call, + # which is what a temporary frame that dropped it would record. + engine = _ClonedCallRecorder(stand_in) + run(base, engine) + assert engine.batches == [1] * calls[stage], stage diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index f430f21b89..9e428bfcfb 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -3412,6 +3412,7 @@ def recording_rules_engine(*, clones: int | None = None) -> _FakeUKEngine: assert household["geography_household_key"].is_unique sidecar = json.loads(output.with_suffix(".build.json").read_text()) assert sidecar["pipeline"] == "uk-frs-spine" + assert sidecar["geographic_clone_count"] == 1 # The build record says which of its stages read the SPI Public Use Tape # (microcosm#1095), from the artifacts the run's spec declares. This # synthetic spec's SPI stage reads a fabricated tab, not the tape. @@ -3690,6 +3691,349 @@ def recorded(compiled, **kwargs): assert len(set(compiled.versions.values())) > 1 +def _driver_arguments(raw_dir: Path, output: Path, spi_tab: Path, hmrc_ods: Path): + return [ + "--frs-raw-dir", + str(raw_dir), + "--spine-h5", + str(output), + "--spi-tab", + str(spi_tab), + "--hmrc-ods", + str(hmrc_ods), + "--no-staging", + ] + + +def test_driver_builds_the_geographic_clone_count_it_is_asked_for( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """``--n-clones K`` (microcosm#931) is the geographic_expand node's + parameter, the clone count of the transform that makes the clones and of + the engine the spine build asks for, which refuses a frame carrying + another count. It never reaches the engine identity: the gates' engine + identity and the provenance in the sidecar's rules engine are the same at + every K, and the rules engine adds the adapter's batch record only at + K > 1. The sidecar records K, and every household comes out K times.""" + + from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine + + pytest.importorskip("tables") + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + tool = _load_tool() + monkeypatch.setattr( + tool, "load_country_spec", lambda country: _synthetic_spec(stage) + ) + asked: list[int | None] = [] + + class _BatchRecordingEngine(_FakeUKEngine): + """The stand-in, with the real adapter's batch record at its clone + count (read from the frame alone; no engine is built).""" + + def __init__(self, clones: int | None) -> None: + self._adapter = PolicyEngineUKEngine(clones=clones) + + def batch_record(self, frame): + return self._adapter.batch_record(frame) + + def recording_rules_engine(*, clones: int | None = None) -> _FakeUKEngine: + asked.append(clones) + return _BatchRecordingEngine(clones) + + monkeypatch.setattr(tool, "_rules_engine", recording_rules_engine) + _stub_policy_readers(monkeypatch) + spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + identities: dict[int, str] = {} + sidecars: dict[int, dict] = {} + households: dict[int, pd.DataFrame] = {} + for clones in (1, 2): + output = tmp_path / f"k{clones}" / "spine.h5" + output.parent.mkdir() + argv = [ + *_driver_arguments(raw_dir, output, spi_tab, hmrc_ods), + "--n-clones", + str(clones), + ] + prepared = tool.prepare_uk_spine_execution(tool.parse_uk_spine_args(argv)) + expand = prepared.graph.node("geographic_expand") + assert expand.params["geographic_clones"] == clones + assert prepared.implementations["geographic_expand"].clones == clones + identities[clones] = prepared.engine_identity + assert tool.main(argv) == 0 + frame, _ = load_uk_national_frame(output) + households[clones] = frame.table("household") + sidecars[clones] = json.loads(output.with_suffix(".build.json").read_text()) + # The preparation and the build each ask for their engine at K. + assert asked == [1, 1, 2, 2] + assert identities[1] == identities[2] + batched = ("engine_batches", "microdf_python") + provenance = { + clones: { + key: value + for key, value in sidecar["rules_engine"].items() + if key not in batched + } + for clones, sidecar in sidecars.items() + } + assert provenance[1] == provenance[2] + assert not set(batched) & set(sidecars[1]["rules_engine"]) + assert sidecars[2]["rules_engine"]["engine_batches"]["batches"] == 2 + for clones, sidecar in sidecars.items(): + assert sidecar["geographic_clone_count"] == clones + household = households[clones] + assert sorted(set(household["household_clone_index"])) == list(range(clones)) + assert len(household) == clones * len(households[1]) + assert set(household["source_household_id"]) == set( + households[1]["household_id"] + ) + + +@pytest.mark.parametrize("value", ["0", "101", "-1", "2.5", "two"]) +def test_driver_refuses_a_clone_count_outside_the_copy_id_fields_bound( + tmp_path: Path, value: str, capsys: pytest.CaptureFixture[str] +) -> None: + tool = _load_tool() + base = _driver_arguments( + tmp_path, tmp_path / "spine.h5", tmp_path / "put2223uk.tab", tmp_path / "x" + ) + with pytest.raises(SystemExit): + tool._parse_args([*base, "--n-clones", value]) + # The refusal names the bound the copy-id field declares. + message = capsys.readouterr().err + assert "--n-clones must be a whole number from 1 to 100" in message + assert tool._parse_args(base).n_clones == 1 + assert tool._parse_args([*base, "--n-clones", "100"]).n_clones == 100 + + +def test_a_release_candidate_spine_takes_one_geographic_clone( + tmp_path: Path, +) -> None: + """A release candidate refuses --n-clones above one until microcosm#931 + group (c) pins the clone count a release builds at: the parser refuses + the pair, and so does the preparation the full build hands a spine request + it has marked a release candidate after parsing it.""" + + tool = _load_tool() + base = _driver_arguments( + tmp_path, tmp_path / "spine.h5", tmp_path / "put2223uk.tab", tmp_path / "x" + ) + with pytest.raises(SystemExit): + tool._parse_args([*base, "--release-candidate", "--n-clones", "2"]) + assert tool._parse_args([*base, "--release-candidate"]).n_clones == 1 + requested = tool._parse_args([*base, "--n-clones", "2"]) + requested.release_candidate = True + with pytest.raises(ValueError, match="builds the spine at one geographic clone"): + tool.prepare_uk_spine_execution(requested) + + +def test_driver_refuses_a_clone_count_its_synthetic_fixture_does_not_make( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A synthetic fixture's transforms and engine clone at its own + config.geographic_clones, so a request for another count is refused while + the spine is prepared, before any stage runs (microcosm#931).""" + + from microcosm.build.uk_runtime import graph_kernels + + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + fixture = tmp_path / "fixture" + fixture.mkdir() + (fixture / "fixture.json").write_text("{}", encoding="utf-8") + tool = _load_tool() + spec = _synthetic_spec(stage) + monkeypatch.setattr(tool, "load_country_spec", lambda country: spec) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) + _stub_policy_readers(monkeypatch) + _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + names = tool._uk_spine_stage_names(spec) + monkeypatch.setattr( + graph_kernels, + "fixture_stage_plan_inputs", + lambda source: ( + tuple(SimpleNamespace(stage=name) for name in names), + {"geographic_expand": SimpleNamespace(clones=1)}, + ), + ) + args = tool.parse_uk_spine_args( + [ + "--synthetic-fixture-dir", + str(fixture), + "--spine-h5", + str(tmp_path / "spine.h5"), + "--smoke", + "--no-staging", + "--n-clones", + "2", + ] + ) + with pytest.raises(ValueError, match="config.geographic_clones"): + tool.prepare_uk_spine_execution(args) + + +def test_driver_profiles_every_node_under_its_id_without_moving_a_key( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """``--stage-profile-dir`` (the memory guard, microcosm#931) runs every + kernel through spine_build's profile proxy, which records each node's RSS + and wall time under the node's id. The proxy keys as the kernel it wraps, + so a profiled run computes every node under the unprofiled run's key, and + a profiled run over the unprofiled run's store finds every node there and + runs, and records, none.""" + + from microcosm.graph import ContentStore, compile_graph, run_graph + + raw_dir = tmp_path / "raw" + stage = _write_fixture(raw_dir) + tool = _load_tool() + monkeypatch.setattr( + tool, "load_country_spec", lambda country: _synthetic_spec(stage) + ) + monkeypatch.setattr(tool, "_rules_engine", _fake_rules_engine) + _stub_policy_readers(monkeypatch) + spi_tab, hmrc_ods = _patch_spi_spine_driver_runtime(tool, monkeypatch, tmp_path) + base = _driver_arguments(raw_dir, tmp_path / "spine.h5", spi_tab, hmrc_ods) + + def prepared(*extra: str): + return tool.prepare_uk_spine_execution( + tool.parse_uk_spine_args([*base, *extra]) + ) + + def run(execution, store: Path, resume: str): + return run_graph( + compile_graph(execution.graph), + sources=execution.sources, + store=ContentStore(store), + kernels=execution.kernels, + resume=resume, + decisions=(), + ) + + plain = prepared() + profile = tmp_path / "profile" + profiled = prepared("--stage-profile-dir", str(profile)) + assert profiled.kernels.refs() == plain.kernels.refs() + assert {type(kernel) for kernel in profiled.kernels.as_mapping().values()} == { + tool._ProfiledKernel + } + assert tool._ProfiledKernel not in { + type(kernel) for kernel in plain.kernels.as_mapping().values() + } + unprofiled = run(plain, tmp_path / "plain", "forbid") + measured = run(profiled, tmp_path / "profiled", "forbid") + keys = {node: receipt.key for node, receipt in unprofiled.nodes.items()} + assert {node: receipt.key for node, receipt in measured.nodes.items()} == keys + records = json.loads((profile / "stage_profile.json").read_text())["stages"] + assert set(records) == set(compile_graph(profiled.graph).order) == set(keys) + assert {record["status"] for record in records.values()} == {"succeeded"} + assert all(record["peak_rss_bytes"] > 0 for record in records.values()) + again = tmp_path / "again" + served = run( + prepared("--stage-profile-dir", str(again)), tmp_path / "plain", "require" + ) + assert all(receipt.hit for receipt in served.nodes.values()) + assert {node: receipt.key for node, receipt in served.nodes.items()} == keys + assert not (again / "stage_profile.json").exists() + + +def _validation_arguments(tmp_path: Path) -> list[str]: + """Driver arguments whose inputs exist, for the argument checks alone.""" + + raw_dir = tmp_path / "raw" + raw_dir.mkdir(exist_ok=True) + spi_tab = tmp_path / "put2223uk.tab" + spi_tab.write_text("", encoding="utf-8") + hmrc_ods = tmp_path / "hmrc.ods" + hmrc_ods.write_bytes(b"") + return _driver_arguments(raw_dir, tmp_path / "spine.h5", spi_tab, hmrc_ods) + + +def test_driver_refuses_a_stage_profile_dir_before_any_node_runs( + tmp_path: Path, +) -> None: + """The profiler adds a node's record only once the node has run, so a + ``--stage-profile-dir`` it could not add to is refused with the + arguments, before any node runs, in a fixed sentence; so is one inside + the repository, since a record keeps the message of a failure its node + raised (microcosm#931).""" + + from microcosm.build.stage_profile import STAGE_PROFILE_SCHEMA_VERSION + + tool = _load_tool() + base = _validation_arguments(tmp_path) + + def validated(directory: Path) -> None: + tool._validate_args( + tool._parse_args([*base, "--stage-profile-dir", str(directory)]) + ) + + # A new directory, and one holding a profile of this schema, are taken. + validated(tmp_path / "new") + kept = tmp_path / "kept" + kept.mkdir() + (kept / "stage_profile.json").write_text( + json.dumps({"schema_version": STAGE_PROFILE_SCHEMA_VERSION, "stages": {}}), + encoding="utf-8", + ) + validated(kept) + + a_file = tmp_path / "a-file" + a_file.write_text("", encoding="utf-8") + with pytest.raises(ValueError) as refused: + validated(a_file) + assert str(refused.value) == "--stage-profile-dir must name a directory." + for name, text in ( + ("unparsed", "{not json"), + ("other-schema", json.dumps({"schema_version": 0, "stages": {}})), + ): + other = tmp_path / name + other.mkdir() + (other / "stage_profile.json").write_text(text, encoding="utf-8") + with pytest.raises(ValueError) as refused: + validated(other) + assert str(refused.value) == ( + "--stage-profile-dir holds a stage_profile.json that is not a stage " + f"profile of schema version {STAGE_PROFILE_SCHEMA_VERSION}, so no " + "node's record could be added to it." + ) + # Named only: nothing is written there. + inside = tool._REPOSITORY / "stage-profile-inside-the-repository" + with pytest.raises(ValueError, match="must lie outside the repository"): + validated(inside) + assert not inside.exists() + + +def test_the_logbook_identity_takes_a_clone_count_other_than_one( + tmp_path: Path, +) -> None: + """A spine of more geographic clones is another build, so its clone count + joins the Logbook identity digest; a one-clone build's digest is hashed + from what it always was (microcosm#931).""" + + tool = _load_tool() + base = _driver_arguments( + tmp_path, tmp_path / "spine.h5", tmp_path / "put2223uk.tab", tmp_path / "x" + ) + configs = { + clones: tool._logbook_run_config( + tool._parse_args([*base, "--n-clones", str(clones)]), + stage_names=("frs_spine", "geographic_expand"), + artifact_pins_digest="f" * 64, + synthetic_fixture=None, + ) + for clones in (1, 2) + } + assert set(configs[1]) == { + "pipeline", + "stages", + "artifact_pins_digest", + "spine_h5", + "synthetic_fixture", + } + assert configs[2] == {**configs[1], "geographic_clones": 2} + + def test_driver_refuses_missing_spi_tab(tmp_path: Path) -> None: tool = _load_tool() raw_dir = tmp_path / "raw" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 6036f7c92a..89d9ab9f15 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -1204,6 +1204,14 @@ def changed_module_bytes(path): "returns its argument unchanged and refuses anything but person, " "benunit or household; the support channel's column names call it" ), + ( + "microcosm.build.uk_runtime.graph_national", + "refuse_clone_first_national_spine", + ): ( + "reads only whether the bound spine's household table carries the " + "geographic clone index, and refuses the spine if it does; the " + "national problem calls it before it resolves anything (microcosm#931)" + ), ("microcosm.frame.rules", "assert_rules_engine_country"): ( "reads the engine's declared country and refuses one that declares " "none or another; the stages that take an engine call it before " @@ -1289,7 +1297,8 @@ def changed_module_bytes(path): "a frame without it; their population is the expand's, so their key " "chains on it too. The national problem resolves its measures in one " "block, which never reads it (full_measure._engine_block_frames; " - "checked)" + "checked); its early refusal of a clone-first spine reads it only " + "through refuse_clone_first_national_spine, a refusal-only helper" ), } #: Where the kernel rule stops, each with why a kernel that names the @@ -2315,6 +2324,35 @@ def test_the_kernel_rule_sees_every_kernel_the_uk_runtime_defines() -> None: assert sum(map(len, listed)) == len(set().union(*listed)) +def test_the_spine_builds_profile_proxy_is_no_kernel_class() -> None: + """``--stage-profile-dir`` wraps every spine kernel in spine_build's + private profile proxy, the memory guard (microcosm#931). The proxy + declares no ``ref`` of its own, so the census of kernel classes above + does not count it, and the rule holds the kernel it wraps. It forwards + that kernel's reference and capabilities, and its bound + ``implementation_hash``, so it hashes exactly what the kernel hashes and + its nodes keep their keys (test_uk_frs_spine runs the driver both ways).""" + from microcosm.build.uk_runtime import spine_build + from microcosm.graph import Kernel, KernelRegistry + + proxy_class = spine_build._ProfiledKernel + assert proxy_class.__module__ == spine_build.__name__ + defined = kernel_classes_defined( + [(spine_build.__name__, Path(spine_build.__file__))] + ) + assert proxy_class not in defined + for ref in ("uk.create@1", "uk.stage.frs_brma@1", "uk.spine-gates@1"): + kernel = _uk_graph_kernels()[ref] + proxy = proxy_class(kernel, Path("unused")) + assert isinstance(proxy, Kernel) + assert KernelRegistry().register(proxy) is proxy + assert proxy.ref == kernel.ref + assert proxy.capabilities is kernel.capabilities + assert proxy.implementation_hash == kernel.implementation_hash + assert proxy.implementation_hash() == kernel.implementation_hash() + assert _hashed_modules(proxy) == _hashed_modules(kernel), ref + + def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: """The stage rule, for every other UK graph kernel (microcosm#931): each module holding code a kernel's own methods can run to compute a value is @@ -2369,9 +2407,9 @@ def test_every_uk_graph_kernel_hashes_each_helper_it_computes_through() -> None: assert population["population"].default == "uk.full.expand" # And the premises of their reason for the writers. On the spine, the # geographic expansion alone writes the clone index columns, and its key - # hashes geographic_clones; the spine build installs it at one clone. The - # dense expand refuses a spine carrying them - # (test_dense_refuses_a_clone_first_spine_at_any_clone_count). + # hashes geographic_clones; the spine build installs it at the clone count + # its --n-clones asks for, one by default. The dense expand refuses a spine + # carrying them (test_dense_refuses_a_clone_first_spine_at_any_clone_count). from microcosm.build.uk_runtime import spine_build from microcosm.build.uk_runtime.graph import uk_registry diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 8fccd129d3..3c2d091cd5 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -41,6 +41,7 @@ UKETBServicesStageTransform, allocate_nhs_by_age_gender, ) +from microcosm.build.uk_runtime.frs_take_up import UKTakeUpPopulationPolicy from microcosm.build.uk_runtime.household_lineage import ( HIGH_VALUE_SUPPORT_COPY_COLUMN, HIGH_VALUE_SUPPORT_INDEX_COLUMN, @@ -1493,6 +1494,19 @@ def test_exact_multiples_of_60_receipt_green_at_any_clone_count( assert receipt["copies"] == split.copies_created == clone_count * 259 assert exact == clone_count * 6 - (clone_count in (7, 13)) assert within == 0 + # The exact recount from the re-summed fold counts the 7,620 family + # one copy more in each clone the comment above names, and in no + # other family or clone: copies_recomputed - copies. + assert one_receipt["copies_recomputed"] == one_receipt["copies"] + assert ( + receipt["copies_recomputed"] - receipt["copies"] + == { + 7: 0, + 11: 11, + 13: 12, + 15: 0, + }[clone_count] + ) @pytest.mark.parametrize("clone_count", [1, 3, 11]) def test_a_count_moved_off_a_non_multiple_is_reported( @@ -3033,6 +3047,59 @@ def test_the_tool_entry_point_receipts_e5( assert tool.main() == (1 if tamper else 0) assert json.loads(output.read_text())["matches_stored_columns"] is not tamper + def test_the_reversals_follow_the_carriers_the_artifact_keeps(self) -> None: + from microcosm.build.uk_runtime.regional_home_values import RANK_BASIS_COLUMN + + tool = _load_tool() + final = _home_values_chain(cgt=False).final + assert tool._reversals_applied(final, "was_wealth") == ("regional_home_values",) + released = _with_tables( + final, household=final.table("household").drop(columns=[RANK_BASIS_COLUMN]) + ) + assert tool._reversals_applied(released, "was_wealth") == () + # A stage after both reversed stages reverses neither. + assert tool._reversals_applied(final, "isa_subscriptions") == () + + def test_the_entry_point_reports_the_rewrite_an_artifact_cannot_reverse( + self, monkeypatch, tmp_path + ) -> None: + # An artifact without regional_home_values' rank basis, a release + # say, keeps that stage's values, which the scope cannot take back: + # the entry point passes E5 only the reversals the scope applied, so + # both property columns are reported as rewritten later rather than + # held to values the scope did not restore. + import json + import sys + + from microcosm.build.uk_runtime.regional_home_values import RANK_BASIS_COLUMN + + tool = _load_tool() + final = _home_values_chain(cgt=False).final + released = _with_tables( + final, household=final.table("household").drop(columns=[RANK_BASIS_COLUMN]) + ) + monkeypatch.setattr(tool, "load_uk_national_frame", lambda path: (released, {})) + output = tmp_path / "receipt.json" + monkeypatch.setattr( + sys, + "argv", + [ + "verify_uk_identity_stability.py", + "--input-h5", + str(tmp_path / "spine.h5"), + "--output", + str(output), + "--check", + "e5", + ], + ) + assert tool.main() == 0 + receipt = json.loads(output.read_text()) + assert set(receipt["rewritten_later"]["household"]) == { + "main_residence_value", + "property_wealth", + } + def test_the_reversal_restores_what_stage_27_replaced(self) -> None: from microcosm.build.uk_runtime.regional_home_values import RANK_BASIS_COLUMN @@ -3205,3 +3272,915 @@ def test_the_stacking_map_names_every_structural_stage_flag() -> None: "frs_brma", ) ) + + +# The clone count every receipt reads, the columns later stages rewrite, and +# the frame E4's stages drew on (microcosm#931). Every frame below is +# synthetic. + + +def _sidecar(clones: int, *, split_clones: int | None = None, **extra) -> dict: + """A spine sidecar's clone records: the build's count and, unless + ``split_clones`` is None, a cgt_support_split receipt counting on it.""" + + sidecar: dict[str, object] = { + "geographic_clone_count": clones, + "stages": ["frs_spine", "geographic_expand"], + "stage_evidence": {}, + } + if split_clones is not None: + sidecar["stages"].append("cgt_support_split") + sidecar["stage_evidence"]["cgt_support_split"] = { + "parameters": {"clone_count": split_clones} + } + sidecar.update(extra) + return sidecar + + +class TestGeographicCloneCounts: + def test_the_frame_carries_its_clone_count(self) -> None: + tool = _load_tool() + assert tool.geographic_clone_counts(_frame()) == {"frame": 1} + _, three = _clone_first_split_then_cloned(3) + assert tool.geographic_clone_counts(three) == {"frame": 3} + assert tool.geographic_clone_counts(three, sidecar=_sidecar(3)) == { + "frame": 3, + "sidecar": 3, + "cgt_support_split": None, + } + assert tool.geographic_clone_counts( + three, sidecar=_sidecar(3, split_clones=3) + ) == {"frame": 3, "sidecar": 3, "cgt_support_split": 3} + + @pytest.mark.parametrize( + ("sidecar", "message"), + [ + ( + _sidecar(1), + "The frame's geographic clone count and the spine sidecar's are " + "not one count (frame 3, sidecar 1); every receipt reads one " + "clone count (microcosm#931).", + ), + ( + _sidecar(3, split_clones=1), + "The frame's geographic clone count, the spine sidecar's and the " + "one cgt_support_split counted its copies on are not one count " + "(frame 3, sidecar 3, cgt_support_split 1); every receipt reads " + "one clone count (microcosm#931).", + ), + ], + ) + def test_a_count_that_differs_is_refused_by_count( + self, sidecar: dict, message: str + ) -> None: + tool = _load_tool() + _, three = _clone_first_split_then_cloned(3) + with pytest.raises(ValueError) as caught: + tool.geographic_clone_counts(three, sidecar=sidecar) + assert str(caught.value) == message + + @pytest.mark.parametrize( + ("sidecar", "message"), + [ + ( + {"stages": []}, + "The spine sidecar records no geographic_clone_count, so the " + "frame's geographic clone count cannot be checked against the " + "spine's (microcosm#931).", + ), + ( + {"geographic_clone_count": True, "stages": []}, + "The spine sidecar records no geographic_clone_count, so the " + "frame's geographic clone count cannot be checked against the " + "spine's (microcosm#931).", + ), + ( + {"geographic_clone_count": 0, "stages": []}, + "The spine sidecar records no geographic_clone_count, so the " + "frame's geographic clone count cannot be checked against the " + "spine's (microcosm#931).", + ), + ( + {"geographic_clone_count": 1}, + "The spine sidecar records no stage list.", + ), + ( + {"geographic_clone_count": 1, "stages": ["cgt_support_split"]}, + "The spine sidecar's cgt_support_split receipt records no " + "clone_count, so the count it counted its copies on cannot be " + "checked (microcosm#931).", + ), + ], + ) + def test_a_missing_record_is_refused_in_a_fixed_sentence( + self, sidecar: dict, message: str + ) -> None: + tool = _load_tool() + with pytest.raises(ValueError) as caught: + tool.geographic_clone_counts(_frame(), sidecar=sidecar) + assert str(caught.value) == message + + def test_a_sidecar_of_another_file_is_refused(self, tmp_path) -> None: + import hashlib + + tool = _load_tool() + spine = tmp_path / "spine.h5" + spine.write_bytes(b"synthetic spine bytes") + named = {"sha256": hashlib.sha256(spine.read_bytes()).hexdigest()} + assert tool.geographic_clone_counts( + _frame(), sidecar=_sidecar(1, output=named), input_h5=spine + ) == {"frame": 1, "sidecar": 1, "cgt_support_split": None} + other = {"sha256": hashlib.sha256(b"another spine").hexdigest()} + with pytest.raises(ValueError) as caught: + tool.geographic_clone_counts( + _frame(), sidecar=_sidecar(1, output=other), input_h5=spine + ) + assert str(caught.value) == ( + "The spine sidecar records another H5 file than --input-h5: its " + "output digest differs." + ) + + @pytest.mark.parametrize("recorded", [3, 2]) + def test_the_entry_point_checks_the_sidecar_before_any_receipt( + self, recorded: int, monkeypatch, tmp_path + ) -> None: + import json + import sys + + tool = _load_tool() + _, three = _clone_first_split_then_cloned(3, _NESTING_SCENARIO) + monkeypatch.setattr(tool, "load_uk_national_frame", lambda path: (three, {})) + receipts: list[object] = [] + monkeypatch.setattr( + tool, + "e9_identity_receipt", + lambda frame, **kwargs: ( + receipts.append(frame) + or {"identical_under_permutation": True, "matches_stored_columns": True} + ), + ) + sidecar = tmp_path / "spine.build.json" + sidecar.write_text(json.dumps(_sidecar(recorded, split_clones=3))) + output = tmp_path / "receipt.json" + monkeypatch.setattr( + sys, + "argv", + [ + "verify_uk_identity_stability.py", + "--input-h5", + str(tmp_path / "spine.h5"), + "--output", + str(output), + "--check", + "e9", + "--spine-sidecar", + str(sidecar), + ], + ) + if recorded != 3: + with pytest.raises( + ValueError, match=r"\(frame 3, sidecar 2, cgt_support_split 3\)" + ): + tool.main() + assert receipts == [] and not output.exists() + return + assert tool.main() == 0 + assert len(receipts) == 1 + assert json.loads(output.read_text())["geographic_clones"] == { + "frame": 3, + "sidecar": 3, + "cgt_support_split": 3, + } + + +class TestLaterRewrites: + """A column a later stage declares writing is reported, not compared. + + The map is the packaged manifest's: each recomputed column belongs to the + last of the receipt's stages that declares it, and every stage after that + one that declares it again may have rewritten the value the final frame + holds. + """ + + def test_the_map_follows_the_manifest(self, monkeypatch) -> None: + tool = _load_tool() + stages = tool._source_stage_roster() + names = [stage["stage"] for stage in stages] + mapped = tool._later_writers( + ["would_claim_uc", "would_claim_uc_childcare", "student_loan_balance"], + checked_stages=(*tool._E4_STAGES, *tool._E5_STAGES), + ) + for column, writers in mapped.items(): + own = max( + position + for position, stage in enumerate(stages) + if stage["stage"] in (*tool._E4_STAGES, *tool._E5_STAGES) + and column + in {*(stage.get("outputs") or ()), *(stage.get("rewrites") or ())} + ) + assert writers == tuple( + stage["stage"] + for stage in stages[own + 1 :] + if column + in {*(stage.get("outputs") or ()), *(stage.get("rewrites") or ())} + ) + assert "would_claim_uc_childcare" not in mapped + assert mapped["student_loan_balance"] == ("student_loans",) + assert {"uc_take_up", "uc_capital_coherence"} <= set(mapped["would_claim_uc"]) + + # A rewrite the manifest gains is reported; one the scope took back is + # not; a column no checked stage declares is refused. + edited = [dict(stage) for stage in stages] + last = edited[names.index("trading_allowance_receipts")] + last["rewrites"] = [*(last.get("rewrites") or ()), "would_claim_uc_childcare"] + monkeypatch.setattr(tool, "_source_stage_roster", lambda: edited) + assert tool._later_writers( + ["would_claim_uc_childcare"], checked_stages=tool._E4_STAGES + ) == {"would_claim_uc_childcare": ("trading_allowance_receipts",)} + assert ( + tool._later_writers( + ["would_claim_uc_childcare"], + checked_stages=tool._E4_STAGES, + reversed_stages=("trading_allowance_receipts",), + ) + == {} + ) + with pytest.raises(ValueError, match="cannot place it"): + tool._later_writers(["brma"], checked_stages=tool._E5_STAGES) + + def test_e5_reports_the_balance_student_loans_re_split(self) -> None: + from microcosm.build.uk_runtime.student_loans import assign_student_loan_plans + from microcosm.build.uk_runtime.was_wealth import ( + allocate_student_loan_balance_to_people, + ) + from test_support.microcosm_build.uk_student_loans import ( + _frame as student_loan_frame, + ) + from test_support.microcosm_build.uk_student_loans import _stocks + + tool = _load_tool() + # was_wealth splits each household's balance among its likely holders + # (two graduates share the first household's), and student_loans then + # moves it to its borrowers (the one who reports an outstanding loan). + frame = student_loan_frame( + ages=[30, 32, 41], + repayments=[0.0, 0.0, 0.0], + households=[1, 1, 2], + reports_outstanding=[True, False, True], + age_left=[21, 21, 21], + ) + person = frame.table("person").copy() + person["student_loan_balance"] = allocate_student_loan_balance_to_people( + household_balances=pd.Series([20_000.0, 9_000.0]), + household_ids=pd.Series([1, 2]), + person=person, + ) + wealth = uk_national_frame( + person=person, + benunit=frame.table("benunit"), + household=frame.table("household"), + time_period="2024", + household_weights=frame.weights_for("household").values, + ) + final = assign_student_loan_plans( + wealth, stocks=_stocks(plan_2=0.0, plan_5=0.0, year=2024), year=2024 + ).frame + moved = final.table("person")["student_loan_balance"].to_numpy() + assert not np.allclose(moved, person["student_loan_balance"].to_numpy()) + + receipt = tool.e5_identity_receipt(final, permutation_seed=7) + assert receipt["identical_under_permutation"] is True + assert receipt["matches_stored_columns"] is True + assert receipt["rewritten_later"] == { + "person": {"student_loan_balance": ["student_loans"]} + } + # Held to the store, as the tool before held it, the balance is red, + # and on the frame was_wealth left it is green. + held = tool.e5_identity_receipt(final, permutation_seed=7, rewritten_later={}) + assert held["stored_column_mismatches"] == {"person": ["student_loan_balance"]} + before = tool.e5_identity_receipt( + wealth, permutation_seed=7, rewritten_later={} + ) + assert before["matches_stored_columns"] is True + + +# E4 on a frame the stages built. Sixteen synthetic households in two regions: +# a claimant in each, aged 22 in full-time work in the odd ones (the apprentice +# walk's population) and 40 in part-time work in the even ones, a child in +# every fourth and a pension-age member, in a +# benefit unit of their own, in every third. They run through +# geographic_expand, frs_take_up, frs_person_draws, frs_household_draws and +# frs_brma (on an engine whose LHA_category reads a claim to Universal Credit, +# would_claim_uc or one at legacy closure, as policyengine-uk's does through +# uc_non_dependants_counted), the SPI support channel at the manifest's shares +# and the income band donors. Then a later stage rewrites would_claim_uc and +# uc_legacy_closure_take_up first writes its claim, as their declarations say +# they do (both drawn here from keyed uniforms). + +_E4_POLICY = UKTakeUpPopulationPolicy( + adult_age=18, state_pension_age=66, instant="2024-01-01", source="test" +) +_E4_BRMA_COUNTS = { + "cells": { + region: { + "A": {f"{region}_A1": 3, f"{region}_A2": 1}, + "B": {f"{region}_B1": 1, f"{region}_B2": 2}, + } + for region in ("LONDON", "SOUTH_EAST") + } +} + + +class _E4Contract: + """Take-up rates for the scenario; the apprentice rate puts the walk's + target where the final frame's weights move it to another prefix.""" + + build_year = 2024 + rates = { + "child_benefit": 0.5, + "child_benefit_opts_out_rate": 0.1, + "pension_credit": 0.5, + "universal_credit": 0.5, + "tax_free_childcare": 0.5, + "extended_childcare": 0.5, + "extended_childcare_age_0_usage": 0.5, + "extended_childcare_age_1_usage": 0.5, + "universal_childcare": 0.5, + "targeted_childcare": 0.5, + "uc_childcare_single": 0.5, + "uc_childcare_couple": 0.5, + "marriage_allowance": 0.5, + "scp_under_6": 0.97, + "scp_6_plus": 0.85, + "tv_ownership_rate": 0.5, + "tv_licence_evasion_rate": 0.5, + "first_time_buyer_rate": 0.5, + "property_purchase_rate": 0.5, + "tax_free_childcare_spend_routed_share": 0.593, + "apprentice_stock_19_to_24": 0.43, + } + + def rate(self, key: str, build_year: int | None = None) -> float: + return self.rates[key] + + def continuous_entry(self, key: str): + return {"mean": 15.019, "sd": 4.972, "lower": 0, "upper": 30} + + def entry(self, key: str): + return SimpleNamespace(raw={"entity": "person"}) + + +class _LHAEngine: + """LHA_category by whether the benefit unit claims Universal Credit.""" + + country = "uk" + + def __init__(self, *, clones: int | None = None) -> None: + self.clones = clones + self.frames: list[object] = [] + + def materialize(self, frame, variables, period): + assert tuple(variables) == ("LHA_category",) + self.frames.append(frame) + benunit = frame.table("benunit") + claims = benunit["would_claim_uc"].to_numpy(dtype=bool) + if "would_claim_uc_at_legacy_closure" in benunit.columns: + claims = claims | benunit["would_claim_uc_at_legacy_closure"].to_numpy( + dtype=bool + ) + return {"LHA_category": np.where(claims, "A", "B")} + + +def _manifest_stage(name: str): + from microcosm.build.country_spec import load_country_spec + + return load_country_spec("uk").sources.stage_map()[name] + + +def _e4_raw_frame(n_households: int = 16): + people = [] + for h in range(1, n_households + 1): + unit = h * 100 + 1 + people.append( + { + "person_id": h * 1000 + 1, + "person_benunit_id": unit, + "age": 22 if h % 2 else 40, + "employment_status": "FT_EMPLOYED" if h % 2 else "PT_EMPLOYED", + "is_uc_claimant": True, + "gender": "MALE" if h % 3 else "FEMALE", + "universal_credit_reported": 10.0 if h % 5 == 0 else 0.0, + "pension_credit_reported": 0.0, + "child_benefit_reported": 5.0 if h % 4 == 0 else 0.0, + } + ) + if h % 4 == 0: + people.append( + { + "person_id": h * 1000 + 2, + "person_benunit_id": unit, + "age": 3, + "employment_status": "CHILD", + "is_uc_claimant": False, + "gender": "FEMALE", + } + ) + if h % 3 == 0: + people.append( + { + "person_id": h * 1000 + 3, + "person_benunit_id": h * 100 + 2, + "age": 70, + "employment_status": "RETIRED", + "is_uc_claimant": True, + "gender": "FEMALE", + "pension_credit_reported": 4.0 if h % 6 == 0 else 0.0, + } + ) + person = pd.DataFrame(people).fillna( + { + "universal_credit_reported": 0.0, + "pension_credit_reported": 0.0, + "child_benefit_reported": 0.0, + } + ) + person["person_household_id"] = person["person_benunit_id"] // 100 + person["months_since_last_birthday"] = 0.0 + for column in ("person_id", "person_benunit_id", "person_household_id"): + person[column] = person[column].astype("int64") + ids = np.arange(1, n_households + 1, dtype="int64") + return uk_national_frame( + person=person.sort_values("person_id").reset_index(drop=True), + benunit=pd.DataFrame( + { + "benunit_id": np.sort(person["person_benunit_id"].unique()), + "is_married": False, + } + ), + household=pd.DataFrame( + { + "household_id": ids, + "region": np.where(ids % 2 == 0, "SOUTH_EAST", "LONDON"), + } + ), + household_weights=np.full(n_households, 100.0), + weight_kind=WeightKind.DESIGN, + time_period="2024", + ) + + +def _with_tables(frame, **tables): + from microcosm.build.uk_runtime.national_frame import uk_household_weight_kind + + return uk_national_frame( + person=tables.get("person", frame.table("person")), + benunit=tables.get("benunit", frame.table("benunit")), + household=tables.get("household", frame.table("household")), + time_period="2024", + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + + +def _e4_propensity() -> pd.DataFrame: + """Every candidate cell takes every reserved band.""" + + from microcosm.build.uk_runtime.spi_band_donors import ( + SPI_INCOME_BAND_DONOR_LOWER_BOUNDS, + SPI_INCOME_BAND_PROPENSITY_AGE_BANDS, + ) + + return pd.DataFrame( + [ + { + "region": region, + "gender": gender, + "age_band": lower, + "band": band, + "band_weight": 1.0, + "cell_weight": 10.0, + "propensity": 0.1, + } + for region in ("LONDON", "SOUTH_EAST") + for gender in ("MALE", "FEMALE") + for lower, _ in SPI_INCOME_BAND_PROPENSITY_AGE_BANDS + for band in SPI_INCOME_BAND_DONOR_LOWER_BOUNDS + ] + ) + + +def _e4_chain(clones: int) -> SimpleNamespace: + """The scenario through E4's stages and the later layers; each stage's frame.""" + + from microcosm.build.stochastic_assignment import stable_identity_uniforms + from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform + from microcosm.build.uk_runtime.frs_household_draws import ( + add_frs_household_draws, + ) + from microcosm.build.uk_runtime.frs_person_draws import ( + FRS_PERSON_DRAW_OUTPUT_COLUMNS, + derive_frs_person_draws, + uk_person_household_weights, + ) + from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform + from microcosm.build.uk_runtime.geographic_expand import ( + expand_uk_geographic_clones, + ) + from microcosm.build.uk_runtime.receipt_counts import record_source_households + from microcosm.build.uk_runtime.spi_band_donors import ( + stack_spi_income_band_donors, + ) + from microcosm.build.uk_runtime.spi_spine import UKSPISupportChannelStageTransform + + contract = _E4Contract() + expanded = expand_uk_geographic_clones(_e4_raw_frame(), clones=clones).frame + taken = UKFRSTakeUpStageTransform( + contract=contract, + stage=_manifest_stage("frs_take_up"), + population_policy=_E4_POLICY, + )(expanded) + # The person draws through their derivation: the stage transform reads + # the installed engine's apprentice schedule, which only its receipt uses. + person = taken.table("person").copy() + drawn = derive_frs_person_draws( + person, + contract=contract, + person_weights=uk_person_household_weights(taken), + person_sources=record_source_households( + taken, "person", stage="frs_person_draws", before_copies=True + ), + ) + for column in FRS_PERSON_DRAW_OUTPUT_COLUMNS: + person[column] = drawn[column].to_numpy() + households = add_frs_household_draws( + _with_tables(taken, person=person), contract=contract + ) + brma = UKFRSBRMAStageTransform( + stage=_manifest_stage("frs_brma"), + engine=_LHAEngine(), + count_resource=_E4_BRMA_COUNTS, + )(households) + supported = UKSPISupportChannelStageTransform( + stage=_manifest_stage("spi_support_channel"), + state_pension_age=_E4_POLICY.state_pension_age, + )(brma) + donors = stack_spi_income_band_donors( + supported, + propensity=_e4_propensity(), + band_taxpayers={ + 200_000: 50.0, + 500_000: 40.0, + 1_000_000: 30.0, + 2_000_000: 20.0, + }, + minimum_donors=1, + maximum_donor_weight=100.0, + taxpayer_period=2024, + ).frame + benunit = donors.table("benunit").copy() + ids = benunit["benunit_id"].to_numpy() + redrawn = stable_identity_uniforms(ids, seed=5, salt="later_uc_redraw") < 0.5 + claims = benunit["would_claim_uc"].to_numpy(dtype=bool) + benunit["would_claim_uc"] = np.where(redrawn, ~claims, claims) + benunit["would_claim_uc_at_legacy_closure"] = ( + stable_identity_uniforms(ids, seed=6, salt="later_legacy_closure") < 0.3 + ) + return SimpleNamespace( + contract=contract, + brma=brma, + donors=donors, + final=_with_tables(donors, benunit=benunit), + ) + + +def _weights_by_household(frame) -> pd.Series: + return pd.Series( + np.asarray(frame.weights_for("household").values, dtype=float), + index=frame.table("household")["household_id"].to_numpy(), + ) + + +def _e4_receipt(tool, frame, contract, **arguments): + return tool.e4_identity_receipt( + frame, + contract=contract, + count_resource=_E4_BRMA_COUNTS, + permutation_seed=7, + population_policy=_E4_POLICY, + **arguments, + ) + + +class TestE4OnAFrameTheStagesBuilt: + @pytest.mark.parametrize("clones", [1, 3]) + def test_the_entry_point_receipts_green_with_the_engine_at_the_clone_count( + self, clones: int, monkeypatch, tmp_path + ) -> None: + import json + import sys + + from microcosm.build.uk_runtime import take_up_contract + from microcosm.frame.adapters import policyengine_uk + + tool = _load_tool() + built = _e4_chain(clones) + engines: list[_LHAEngine] = [] + + def engine(*, clones=None): + engines.append(_LHAEngine(clones=clones)) + return engines[-1] + + monkeypatch.setattr(policyengine_uk, "PolicyEngineUKEngine", engine) + monkeypatch.setattr( + take_up_contract, "load_uk_take_up_contract", lambda: built.contract + ) + monkeypatch.setattr(tool, "uk_take_up_population_policy", lambda _: _E4_POLICY) + monkeypatch.setattr(tool, "load_brma_count_resource", lambda: _E4_BRMA_COUNTS) + monkeypatch.setattr( + tool, "load_uk_national_frame", lambda path: (built.final, {}) + ) + output = tmp_path / "receipt.json" + monkeypatch.setattr( + sys, + "argv", + [ + "verify_uk_identity_stability.py", + "--input-h5", + str(tmp_path / "spine.h5"), + "--output", + str(output), + "--check", + "e4", + ], + ) + + assert tool.main() == 0 + receipt = json.loads(output.read_text()) + assert receipt["identical_under_permutation"] is True + assert receipt["matches_stored_columns"] is True + assert receipt["stored_mismatches"] == {} + assert receipt["geographic_clones"] == {"frame": clones} + # The engine took the spine's clone count, and read every clone. + assert [engine.clones for engine in engines] == [clones] + (read,) = engines[0].frames + assert read.table("household")["household_clone_index"].nunique() == clones + # The take-up flags later stages rewrite are reported by those stages, + # as the manifest declares them. + rewritten = receipt["rewritten_later"]["benunit"] + assert {"uc_take_up", "uc_capital_coherence"} <= set( + rewritten["would_claim_uc"] + ) + assert rewritten["would_claim_pc"] == ["pension_credit_take_up"] + assert rewritten["would_claim_child_benefit"] == ["child_benefit_take_up"] + assert rewritten["child_benefit_opts_out"] == ["child_benefit_take_up"] + assert "would_claim_uc_childcare" not in rewritten + # Those the store still holds are named: the columns + # childcare_copy_take_up declares, which no stage of the scenario + # rewrites, but not would_claim_uc, which a later stage redrew. + childcare = { + column + for columns in receipt["rewritten_later"].values() + for column, stages in columns.items() + if "childcare_copy_take_up" in stages + } + still_equal = { + column + for columns in receipt["rewritten_later_still_equal"].values() + for column in columns + } + assert childcare and childcare <= still_equal + assert "would_claim_uc" not in still_equal + + def test_brma_on_the_final_frames_flags_draws_from_other_cells(self) -> None: + # The BRMA stage read LHA_category on its own input, carrying + # frs_take_up's would_claim_uc and no legacy closure claim. The final + # frame carries both as later stages left them, so LHA_category + # materialized there puts some benefit units in another cell, and + # their BRMA draws differ, though no later stage declares writing + # brma. On the frame the stage read, as the recompute restores it, + # every draw is the stage's. + tool = _load_tool() + built = _e4_chain(1) + scoped = tool._frame_as_draw_stages_saw( + built.final, state_pension_age=_E4_POLICY.state_pension_age + ) + final_lha = _LHAEngine().materialize( + tool._engine_safe_frame(scoped), ("LHA_category",), "2024" + )["LHA_category"] + on_final = _e4_receipt(tool, scoped, built.contract, lha_category=final_lha) + assert "brma" in on_final["stored_mismatches"]["household"] + + engine = _LHAEngine() + receipt = _e4_receipt(tool, scoped, built.contract, engine=engine) + assert receipt["stored_mismatches"] == {} + assert receipt["matches_stored_columns"] is True + (read,) = engine.frames + benunit = read.table("benunit").set_index("benunit_id") + stage_time = built.brma.table("benunit").set_index("benunit_id") + assert "would_claim_uc_at_legacy_closure" not in benunit.columns + np.testing.assert_array_equal( + benunit["would_claim_uc"].to_numpy(), + stage_time["would_claim_uc"].reindex(benunit.index).to_numpy(), + ) + scope = receipt["lha_category_scope"] + assert scope["stage"] == "frs_brma" + assert "would_claim_uc" in scope["restored_from_the_recompute"] + assert scope["columns_first_written_later"] > 0 + # What the artifact cannot restore is named, with its writers: the + # reported award the scenario carries, which the reporter redraw + # rewrites later. + assert ( + "uc_reporter_redraw" + in scope["carried_as_stored"]["universal_credit_reported"] + ) + + def test_the_apprentice_walk_on_the_final_weights_flags_another_prefix( + self, + ) -> None: + # frs_person_draws walked the eligible employees at geographic_expand's + # weights. The SPI support channel then kept half of a household's + # weight, or four fifths where a member is of pension age, and the + # band donors' funding scaled each region's households again; the + # CGT layers fold back. Nearest the target on the final frame's + # weights lies another prefix, though no later stage declares + # writing is_apprentice. The scope takes both moves back, to rounding. + tool = _load_tool() + built = _e4_chain(1) + stage_weights = _weights_by_household(built.brma) + final_scope = tool._frs_only_frame(built.final) + moved = _weights_by_household(final_scope).reindex(stage_weights.index) + assert len(np.unique(np.round(moved / stage_weights, 12))) > 1 + + on_final = _e4_receipt(tool, final_scope, built.contract, engine=_LHAEngine()) + assert on_final["stored_mismatches"] == {"person": ["is_apprentice"]} + + scoped = tool._frame_as_draw_stages_saw( + built.final, state_pension_age=_E4_POLICY.state_pension_age + ) + np.testing.assert_allclose( + _weights_by_household(scoped).reindex(stage_weights.index).to_numpy(), + stage_weights.to_numpy(), + rtol=1e-12, + ) + receipt = _e4_receipt(tool, scoped, built.contract, engine=_LHAEngine()) + assert receipt["stored_mismatches"] == {} + assert receipt["held_to_tolerance"] == {} + + def test_the_cgt_layers_fold_back_to_the_weights_before_them(self) -> None: + # The #970 anchor moves weight between each clone pair's halves and the + # support split divides families; the scope's folds give the weights + # before both back to rounding, so neither moves the walk. + tool = _load_tool() + _, cloned = _split_then_cloned() + anchored = _through_the_residential_split(cloned) + household = anchored.table("household") + assert household[HOUSEHOLD_IS_CGT_CLONE].any() + scoped = tool._frame_as_draw_stages_saw(anchored, state_pension_age=66) + np.testing.assert_allclose( + scoped.weights_for("household").values, _SPLIT_WEIGHTS, rtol=1e-12 + ) + + def test_a_rewritten_column_the_store_no_longer_holds_is_not_named_equal( + self, + ) -> None: + # childcare_copy_take_up declares writing would_claim_tfc, so E4 does + # not compare it. Flipped on a survey unit, the receipt stays green, + # but no longer names it among the columns the store still holds. + tool = _load_tool() + built = _e4_chain(1) + scoped = tool._frame_as_draw_stages_saw( + built.final, state_pension_age=_E4_POLICY.state_pension_age + ) + receipt = _e4_receipt(tool, scoped, built.contract, engine=_LHAEngine()) + assert "would_claim_tfc" in receipt["rewritten_later_still_equal"]["benunit"] + benunit = scoped.table("benunit").copy() + benunit["would_claim_tfc"] = benunit["would_claim_tfc"].astype(bool) + benunit.loc[0, "would_claim_tfc"] = not benunit.loc[0, "would_claim_tfc"] + flipped = _e4_receipt( + tool, + _with_tables(scoped, benunit=benunit), + built.contract, + engine=_LHAEngine(), + ) + assert flipped["matches_stored_columns"] is True + assert flipped["rewritten_later"]["benunit"]["would_claim_tfc"] == [ + "childcare_copy_take_up" + ] + assert "would_claim_tfc" not in flipped["rewritten_later_still_equal"].get( + "benunit", [] + ) + + def test_a_flag_held_to_the_walk_is_named(self, monkeypatch) -> None: + tool = _load_tool() + built = _e4_chain(1) + scoped = tool._frame_as_draw_stages_saw( + built.final, state_pension_age=_E4_POLICY.state_pension_age + ) + person = scoped.table("person").copy() + flagged = np.flatnonzero(person["is_apprentice"].to_numpy(dtype=bool)) + person.loc[flagged[-1], "is_apprentice"] = False + unflagged = _with_tables(scoped, person=person) + + receipt = _e4_receipt(tool, unflagged, built.contract, engine=_LHAEngine()) + assert receipt["stored_mismatches"] == {"person": ["is_apprentice"]} + monkeypatch.setattr(tool, "_apprentice_walk_holds", lambda *a, **k: True) + receipt = _e4_receipt(tool, unflagged, built.contract, engine=_LHAEngine()) + assert receipt["stored_mismatches"] == {} + assert receipt["held_to_tolerance"] == {"person": ["is_apprentice"]} + + +class TestTheApprenticeWalkToRounding: + """Stored flags are the walk's on weights the stage read to rounding.""" + + @staticmethod + def _people(n: int = 4) -> pd.DataFrame: + return pd.DataFrame( + { + "person_id": np.arange(1, n + 1, dtype="int64"), + "age": [22] * n, + "employment_status": ["FT_EMPLOYED"] * n, + } + ) + + @staticmethod + def _in_draw_order(person: pd.DataFrame) -> np.ndarray: + from microcosm.build.stochastic_assignment import stable_identity_uniforms + + uniforms = stable_identity_uniforms( + person["person_id"].to_numpy(), seed=0, salt="is_apprentice" + ) + return np.argsort(uniforms, kind="stable") + + def test_a_prefix_as_near_the_target_as_the_walks_holds(self) -> None: + # Two employees of weight 1 at a rate of a quarter: the target, 0.5, + # lies as near no one as the first in draw order, and the stage took + # the first. Restored a float heavier, the first now lies just + # further, and the recompute takes no one; the stored flag lies within + # the tolerance of it and holds. + tool = _load_tool() + person = self._people(2) + order = self._in_draw_order(person) + first = np.zeros(2, dtype=bool) + first[order[0]] = True + weights = np.ones(2) + assert tool._apprentice_walk_holds( + person, first, person_weights=weights, rate=0.25 + ) + weights[order[0]] = np.nextafter(1.0, 2.0) + assert tool._apprentice_walk_holds( + person, first, person_weights=weights, rate=0.25 + ) + assert tool._apprentice_walk_holds( + person, np.zeros(2, dtype=bool), person_weights=weights, rate=0.25 + ) + # Both flagged lie a whole weight further: not held. + assert not tool._apprentice_walk_holds( + person, np.ones(2, dtype=bool), person_weights=weights, rate=0.25 + ) + + def test_flags_off_the_walk_do_not_hold(self) -> None: + tool = _load_tool() + person = self._people(4) + order = self._in_draw_order(person) + weights = np.ones(4) + prefix = np.zeros(4, dtype=bool) + prefix[order[:2]] = True + assert tool._apprentice_walk_holds( + person, prefix, person_weights=weights, rate=0.5 + ) + # The second and third in draw order: no threshold flags them alone. + skipped = np.zeros(4, dtype=bool) + skipped[order[1:3]] = True + assert not tool._apprentice_walk_holds( + person, skipped, person_weights=weights, rate=0.5 + ) + # A flagged person the stage could not flag: aged 30. + older = person.copy() + older.loc[order[0], "age"] = 30 + assert not tool._apprentice_walk_holds( + older, prefix, person_weights=weights, rate=0.5 + ) + + def test_the_tolerance_is_relative_to_the_people_aged_21_to_24(self) -> None: + # Four employees weighing 1, 1, a millionth and 1 in draw order, at a + # rate that puts the target at 1.9: the walk takes the first two, a + # tenth off, and a stored prefix of three lies a millionth further + # off. That is no rounding difference (the tolerance is 1e-9 of about + # 3), so it does not hold. + tool = _load_tool() + person = self._people(4) + order = self._in_draw_order(person) + weights = np.ones(4) + weights[order[2]] = 1e-6 + rate = 1.9 / float(weights.sum()) + two = np.zeros(4, dtype=bool) + two[order[:2]] = True + three = np.zeros(4, dtype=bool) + three[order[:3]] = True + assert tool._apprentice_walk_holds( + person, two, person_weights=weights, rate=rate + ) + assert not tool._apprentice_walk_holds( + person, three, person_weights=weights, rate=rate + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_graph.py index b3985440d3..1ec5a689fc 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_national_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_national_graph.py @@ -67,6 +67,7 @@ _cgt_projection, _signing_key, ) +from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame @pytest.fixture(autouse=True) @@ -654,3 +655,64 @@ def test_national_run_config_refuses_an_empty_register(): registry_document={"national_registry": {}}, driver_parameters={}, ) + + +def test_the_national_problem_is_the_first_national_node_to_read_the_frame(): + """The early refusal sits in the first node of the national line that is + given the bound frame's slices: the targets read only their sources, so it + is the problem (microcosm#931).""" + + national = uk_national_graph( + _config(), + spine=bound_spine_graph(seam._frame()), + spine_population="uk.full.spine_checkpoint", + review_date=REVIEW_DATE, + ) + compiled = compile_graph(national.graph) + readers = [node for node in compiled.order if national.graph.node(node).inputs] + assert readers[0] == NATIONAL_PROBLEM_NODE + assert not national.graph.node(NATIONAL_TARGETS_NODE).inputs + + +def test_the_national_line_refuses_a_clone_first_spine_at_one_clone_too( + tmp_path, monkeypatch +): + """Every spine since microcosm#931 group (b) carries the geographic clone + index, and the national line drops it from its export only from group (c) + on, so the problem refuses a bound spine that carries it, at one clone + too, before anything is resolved or solved, in one fixed sentence. A spine + without it is solved as before (the tests above).""" + + pytest.importorskip("tables") + # One clone carries the index too: the refusal must fire at one clone. + frame = clone_first_frame(seam._frame(), 1) + registry = seam._registry() + _patch_target_inputs(monkeypatch, registry) + sources = _bound_inputs(tmp_path, frame) + national = uk_national_graph( + _config(), + spine=bound_spine_graph(frame), + spine_population="uk.full.spine_checkpoint", + review_date=REVIEW_DATE, + ) + with pytest.raises(ValueError) as refused: + graph_national.refuse_clone_first_national_spine(frame) + message = str(refused.value) + assert "'household_clone_index'" in message + assert "group (c)" in message + with pytest.raises(Exception, match="national line refuses a bound spine"): + run_graph( + compile_graph(national.graph), + sources=sources, + store=ContentStore(tmp_path / "store"), + kernels=_registry_for_run(), + ) + # One fixed sentence: the same at any clone count and size, so it carries + # no count; and nothing is refused without the index. + for clones in (2, 3): + with pytest.raises(ValueError) as again: + graph_national.refuse_clone_first_national_spine( + clone_first_frame(seam._frame(), clones) + ) + assert str(again.value) == message + assert graph_national.refuse_clone_first_national_spine(seam._frame()) is None diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_private_school_attendance.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_private_school_attendance.py index 3982bbec89..addb6fe5b9 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_private_school_attendance.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_private_school_attendance.py @@ -345,3 +345,20 @@ def test_the_stage_refuses_drifted_parameters(ranks): assert stored.table("person")[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT].tolist() == ATTENDS evidence = transform.checkpoint_metadata()["evidence"] assert evidence["rank_route"] == PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE + + +def test_the_stage_reads_the_parameters_the_declaration_restates(): + """microcosm#931: the two parameters the stage's restatement reads + are the ones the UK engine-population declaration restates attendance + from. A call that runs clone by clone holds every batch's engine equal to + the adapter's own tree at exactly those paths, so the stage, reading the + adapter's parameters, reads what each batch computed with.""" + from microcosm.frame.adapters.policyengine_uk_population import ( + UK_ENGINE_POPULATION, + ) + + assert set(UK_ENGINE_POPULATION.restated) == {PRIVATE_SCHOOL_ATTENDANCE_OUTPUT} + assert ( + private_school_attendance.PRIVATE_SCHOOL_ATTENDANCE_PARAMETERS + == UK_ENGINE_POPULATION.restated[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT] + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py index 6dabc198a8..be0a227097 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rules_engine_provenance.py @@ -343,3 +343,37 @@ def version(package: str) -> str: assert _identity(spine_build._rules_engine_provenance()) == _identity( {"package": "policyengine-uk", "version": VERSION} ) + + +def test_the_spine_engine_takes_the_clone_count_its_identity_does_not( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # --n-clones reaches the engine the spine build makes, which then refuses + # a frame carrying another clone count, but never the provenance the + # engine identity is hashed from: the identity is the same at every K + # (microcosm#931). + import inspect + import sys + import types + + from microcosm.build.uk_runtime import hmrc_uprating + + _install(monkeypatch, None) + monkeypatch.setitem( + sys.modules, "policyengine_uk", types.ModuleType("policyengine_uk") + ) + monkeypatch.setattr( + hmrc_uprating, "assert_installed_engine_is_pinned", lambda: "fixture" + ) + frame = _final_frame((0, 1)) + identities = set() + for clones in (1, 2, 4): + engine = spine_build._rules_engine(clones=clones) + if clones == 2: + assert engine.simulation_batches(frame) == 2 + else: + with pytest.raises(ValueError, match="clone"): + engine.simulation_batches(frame) + identities.add(_identity(spine_build._rules_engine_provenance())) + assert identities == {_identity({"package": "policyengine-uk", "version": VERSION})} + assert not inspect.signature(spine_build._rules_engine_provenance).parameters diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py index fa19b8d158..a533851548 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_band_donors.py @@ -1147,12 +1147,14 @@ def test_e8_receipt_reconstructs_the_seating_from_ids() -> None: # carriers and the band weights are the stored ones in any row order. tool = _identity_tool() propensity = _propensity() - stacked = stack_spi_income_band_donors( + stage = stack_spi_income_band_donors( _support_frame(200), propensity=propensity, band_taxpayers=TAXPAYERS, taxpayer_period=2024, - ).frame + ) + stacked = stage.frame + evidence = stage.evidence() problems: dict[str, object] = {} receipt = tool._e8_band_donors( stacked, @@ -1163,11 +1165,20 @@ def test_e8_receipt_reconstructs_the_seating_from_ids() -> None: ) assert problems == {} assert receipt["seating_recomputed"] is True - assert receipt["donor_households"] == 27 - assert receipt["seating_scale"] == "capped" + assert receipt["seating_scale"] == evidence["seating_scale"] == "capped" assert receipt["mass"]["declared_factor"] == 1.0 + # The receipt gives the layer in the stage's capped forms (microcosm#931): + # the donors and each band's in rounded distinct source households, and + # the stage's own funding rows, masses and factors withheld, with the + # funding rule's verdict. + assert receipt["donor_households"] == evidence["donor_count"] == rounded_count(27) assert [row["lower_bound"] for row in receipt["bands"]] == list(BANDS) - assert set(receipt["funding"]) == {"LONDON", "SOUTH_EAST"} + assert [row["donor_households"] for row in receipt["bands"]] == [ + row["donor_households"] for row in evidence["bands"] + ] + assert receipt["funding"] == evidence["funding"] + assert [row["stratum"] for row in receipt["funding"]] == ["LONDON", "SOUTH_EAST"] + assert receipt["funding_within_share"] is evidence["funding_within_share"] is True # Without the tape's propensities the layer is checked for structure and # mass only, and the receipt says so. @@ -1606,22 +1617,24 @@ def test_a_clone_without_candidates_is_refused_without_a_count() -> None: lambda clones: pd.array( [None, *clones.to_numpy()[1:].tolist()], dtype="Int64" ), - "1 household row(s) carry no 'household_clone_index'; SPI income band " + "A household row carries no 'household_clone_index'; SPI income band " "donors seat each geographic clone.", ), ( lambda clones: np.r_[[-1, -1], clones.to_numpy()[2:]], - "2 household row(s) carry a negative 'household_clone_index'; SPI " + "A household row carries a negative 'household_clone_index'; SPI " "income band donors seat each geographic clone.", ), ( lambda clones: clones.replace({1: 2}).to_numpy(), - "1 of the 3 geographic clone indices from 0 to the largest are carried " - "by no household; SPI income band donors seat clones 0 to K - 1.", + "A 'household_clone_index' value from 0 to the largest is carried by " + "no household; SPI income band donors seat clones 0 to K - 1.", ), ], ) -def test_a_malformed_clone_index_is_refused_by_count(index, message: str) -> None: +def test_a_malformed_clone_index_is_refused_without_a_count( + index, message: str +) -> None: frame = _clone_first_support_frame(60, 2) household = frame.table("household").copy() household[_CLONE] = index(household[_CLONE]) @@ -2060,16 +2073,21 @@ def test_e8_receipt_replays_the_seating_of_every_clone() -> None: "band_taxpayers": TAXPAYERS, "permutation_seed": 123, } - stacked = stack_spi_income_band_donors( + stage = stack_spi_income_band_donors( _clone_first_support_frame(200, 3), propensity=propensity, **_PUBLISHED - ).frame + ) + stacked = stage.frame + evidence = stage.evidence() problems: dict[str, object] = {} receipt = tool._e8_band_donors(stacked, problems, **arguments) assert problems == {} assert receipt["seating_recomputed"] is True - assert receipt["donor_households"] == 3 * 27 - assert receipt["seating_scale"] == "capped" - assert set(receipt["funding"]) == { + assert receipt["seating_scale"] == evidence["seating_scale"] == "capped" + # Every clone's donors, counted once per survey household and rounded, and + # every clone's regions in the stage's funding rows. + assert receipt["donor_households"] == evidence["donor_count"] + assert receipt["funding"] == evidence["funding"] + assert {row["stratum"] for row in receipt["funding"]} == { f"{region}|{clone}" for region in ("LONDON", "SOUTH_EAST") for clone in range(3) } @@ -2303,11 +2321,19 @@ def test_e8_receipt_reruns_stage_17_on_the_frame_it_wrote(n_clones: int) -> None ) # and the check scales each funding stratum back by the stage's own - # factor, and the stage rerun there reproduces the stored seats, carriers - # and band weights, with the artifact's people in either order and the - # pre-donor people permuted, reading each donor once. - stored = {row["lower_bound"]: row for row in stage_17.band_rows} + # factor, factors = {row["stratum"]: row["factor"] for row in stage_17.funding_rows} + _, reconstructed = tool._band_donor_funding_factors( + scoped.table("household"), scoped.weights_for("household").values + ) + assert reconstructed.to_dict() == pytest.approx(factors, rel=1e-12) + + # and the stage rerun there reproduces the stored seats, carriers and band + # weights, with the artifact's people in either order and the pre-donor + # people permuted, reading each donor once. The receipt gives the layer in + # the stage's capped forms, the exact weights and factors held, never + # written. + evidence = stage_17.evidence() for artifact in (built.artifact, tool._reverse_rows(built.artifact)): problems: dict[str, object] = {} receipt = tool._e8_band_donors( @@ -2319,15 +2345,18 @@ def test_e8_receipt_reruns_stage_17_on_the_frame_it_wrote(n_clones: int) -> None ) assert problems == {} assert receipt["seating_recomputed"] is True - assert receipt["donor_households"] == stage_17.donor_count - assert receipt["funding"] == pytest.approx(factors, rel=1e-12) - for row in receipt["bands"]: - band = stored[row["lower_bound"]] - assert row["donor_households"] == band["donor_households"] - assert row["donor_weight"] == pytest.approx(band["donor_weight"], rel=1e-12) - assert row["weighted_taxpayers"] == pytest.approx( - band["weighted_taxpayers"], rel=1e-12 - ) + assert receipt["donor_households"] == rounded_count( + stage_17.donor_source_households + ) + assert receipt["funding"] == evidence["funding"] + assert receipt["funding_within_share"] is True + assert receipt["bands"] == [ + { + "lower_bound": row["lower_bound"], + "donor_households": rounded_count(row["donor_source_households"]), + } + for row in stage_17.band_rows + ] def test_e8_receipt_reads_each_donor_through_its_later_copies() -> None: @@ -2431,7 +2460,139 @@ def test_the_e8_entry_hands_the_band_donor_check_the_artifact(monkeypatch) -> No assert receipt["pre_salary_sacrifice_restored"] is True assert receipt["stored_column_mismatches"] == {} assert receipt["band_donors"]["seating_recomputed"] is True - assert receipt["band_donors"]["donor_households"] == stage_17.donor_count + assert receipt["band_donors"]["donor_households"] == rounded_count( + stage_17.donor_source_households + ) + + +# E8's funding rule on frames the stage funds at the floor (microcosm#931). +# Every frame below is synthetic. + + +def _floor_funded_frame(n_households: int, salt: str): + """``_support_frame``'s households, LONDON's light but for one heavy + household and the other region's carrying most of the mass, so the mass + scale binds in LONDON, which keeps exactly one less the reallocation + share; the stage then makes the whole total exact on one incumbent row.""" + + from microcosm.build.stochastic_assignment import stable_identity_uniforms + + frame = _support_frame(n_households) + household = frame.table("household") + uniforms = stable_identity_uniforms( + household["household_id"].to_numpy(), seed=11, salt=salt + ) + london = household["region"].astype(str).eq("LONDON").to_numpy() + weights = np.where(london, 1.0 + uniforms, 9_000.0 + 500.0 * uniforms) + heavy = np.flatnonzero(london)[0] + weights[heavy] = 20_000.0 + 100.0 * uniforms[heavy] + return uk_national_frame( + person=frame.table("person"), + benunit=frame.table("benunit"), + household=household, + time_period="2024", + weight_kind=WeightKind.IMPORTANCE, + household_weights=weights, + ) + + +def _floor_funded_stage(): + stage = stack_spi_income_band_donors( + _floor_funded_frame(900, "j"), + propensity=_propensity(), + band_taxpayers=TAXPAYERS, + taxpayer_period=2024, + ) + floor = 1.0 - spi_band_donors.SPI_INCOME_BAND_MAXIMUM_REALLOCATION_SHARE + assert stage.mass_scale < 1.0 + factors = {row["stratum"]: row["factor"] for row in stage.funding_rows} + assert factors["LONDON"] == floor + assert stage.funding_within_share is True + return stage, floor + + +def _e8_funding(tool, frame) -> tuple[dict[str, object], dict[str, object]]: + problems: dict[str, object] = {} + receipt = tool._e8_band_donors( + frame, problems, propensity=None, band_taxpayers=None, permutation_seed=123 + ) + return receipt, problems + + +def test_e8_holds_the_funding_of_a_stratum_the_stage_funds_at_the_floor() -> None: + """The factor E8 rebuilds from the stored weights carries the stage's + exact-total correction wherever it landed, so E8 holds the funding by + the stage's rule at its own weight tolerance, not at the stage gate's + few ulps: the stage's output is green.""" + + tool = _identity_tool() + stage, _ = _floor_funded_stage() + receipt, problems = _e8_funding(tool, stage.frame) + assert problems == {} + assert receipt["funding_within_share"] is True + + +def test_e8_holds_a_floor_factor_moved_by_rounding_and_refuses_one_underfunded(): + """A floor stratum's incumbent moved down by a trillionth of the + stratum's mass, which an exact-total correction can do to a small + stratum, takes its rebuilt factor beyond the stage gate's ulps but not + E8's tolerance, and is held; moved by a millionth, it is a stratum kept + below one less the share, and E8 says so. The other region's incumbent + takes the mass up, so the total and its mass record stand.""" + + tool = _identity_tool() + stage, floor = _floor_funded_stage() + household = stage.frame.table("household") + weights = np.asarray(stage.frame.weights_for("household").values, dtype=float) + london = household["region"].astype(str).eq("LONDON").to_numpy() + incumbent = ~household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR].to_numpy(dtype=bool) + in_london = np.flatnonzero(london & incumbent) + heaviest = in_london[np.argmax(weights[in_london])] + receiver = np.flatnonzero(~london & incumbent)[0] + stratum_mass = float(weights[london].sum()) + for relative, held in ((1e-12, True), (1e-6, False)): + moved = weights.copy() + moved[heaviest] -= relative * stratum_mass + moved[receiver] += relative * stratum_mass + frame = _reweighted(stage.frame, moved) + _, rebuilt = tool._band_donor_funding_factors( + frame.table("household"), frame.weights_for("household").values + ) + assert float(rebuilt["LONDON"]) < floor * ( + 1.0 - spi_band_donors.SPI_INCOME_BAND_FUNDING_RELATIVE_TOLERANCE + ) + receipt, problems = _e8_funding(tool, frame) + assert receipt["funding_within_share"] is held + assert ("band_donor_funding_share" in problems) is not held + + +def test_e8_refuses_a_stratum_funded_past_its_share() -> None: + tool = _identity_tool() + propensity = _propensity() + stacked = stack_spi_income_band_donors( + _support_frame(200), + propensity=propensity, + band_taxpayers=TAXPAYERS, + taxpayer_period=2024, + ).frame + household = stacked.table("household") + weights = np.asarray(stacked.weights_for("household").values, dtype=float).copy() + incumbent = ~household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR].to_numpy(dtype=bool) + london = household["region"].astype(str).eq("LONDON").to_numpy() + weights[incumbent & london] *= 0.05 + problems: dict[str, object] = {} + receipt = tool._e8_band_donors( + _reweighted(stacked, weights), + problems, + propensity=propensity, + band_taxpayers=TAXPAYERS, + permutation_seed=123, + ) + assert receipt["funding_within_share"] is False + assert problems["band_donor_funding_share"] == ( + "a funding stratum's incumbents kept less than one less the declared " + "reallocation share of its mass" + ) # The stage gate on a capped seating (microcosm#931). A capped receipt gives diff --git a/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py b/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py new file mode 100644 index 0000000000..519d986ddc --- /dev/null +++ b/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py @@ -0,0 +1,956 @@ +"""The clone-first UK spine at four geographic clones on the H2 fixture. + +microcosm#931 group (b). The H2 fixture's invented sources run the whole +spine in this process twice, through the real engine: at one geographic +clone, the fixture's own config, and at four, the same config with +``geographic_clones`` 4, so the graph's geographic_expand node, the transform +that makes the clones and the engine all take K = 4. Each run records, per +graph node, the simulations each engine call ran, the population's mass, its +clone-index invariant and each stage's outputs as weighted shares, and keeps +the populations the exact checks compare. + +- **Every stage runs**, and at K = 4 every engine call from geographic_expand + on runs four simulations, one per clone, in as many calls per stage as at + one clone: no stage calls the engine per clone. private_school_attendance + restates attendance over the pool, the batched route. +- **The clone-index invariant** holds at every node's population from + geographic_expand on. +- **Exact:** clone 0 after geographic_expand and geographic_identity is the + one-clone frame on every column, at K x its weight bitwise. Through + frs_brma it differs from K = 1 only where a draw pools over the clones + (frs_take_up's anchored residual, frs_person_draws' apprentice walk) and in + the BRMA of the households those draws move, whose LHA_category the engine + computes from them; so the keyed per-row draws nest. spi_support_channel + copies K = 1's households in clone 0, at nesting weights, and + spi_income_band_donors seats K = 1's households in K = 1's bands with + K = 1's carriers. +- **The band donors' weights:** their mass scale is reported first. The stage + funds every clone's donors at one scale, the smallest any clone's region + allows, and on this fixture's capped rung that scale at K = 4 is below one + clone's. So clone 0's donors weigh K = 1's times the ratio of the two + scales, and each funding stratum's incumbents K = 1's times one ratio of + the stratum's own: K x clone-0 weights nest on one clone's only where the + two scales are equal. Whether the stage takes a scale per clone is open + (microcosm#931). +- **Tolerance classes:** the pooled draws keep their K-invariant receipt + fields; from frs_take_up on, every output every clone carries, at a + weighted share near one clone's, and every node keeps K = 1's household + mass. +- **Receipts:** the clone count in geographic_expand's, spi_support_channel's + and spi_income_band_donors' receipts and, at K = 4 only, the support clips' + and property components'; the geographic fold receipts at the six folding + fits at K = 4, and none at one clone. + +It cannot catch within-batch int32 collisions: ids past 2**31 reach the engine +at K = 1 already, and these households are too few (a licensed build's +per-batch int32 count is that check). Invented data only; assertion messages +give K, stages, columns, counts and booleans, never a weight, a value or an +id. +""" + +from __future__ import annotations + +import json +import math +import shutil +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass +from fractions import Fraction +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime import spi_band_donors, spi_support +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + ladder_clone_index_column, +) +from microcosm.build.uk_runtime.geographic_expand import ( + GEOGRAPHIC_EXPAND_STAGE_NAME, + UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, +) +from microcosm.build.uk_runtime.geographic_identity import ( + GEOGRAPHIC_IDENTITY_STAGE_NAME, +) +from microcosm.build.uk_runtime.graph import uk_registry, uk_spine_graph +from microcosm.build.uk_runtime.graph_evidence import load_spine_stage_artifacts +from microcosm.build.uk_runtime.graph_kernels import fixture_stage_plan_inputs +from microcosm.build.uk_runtime.household_lineage import GEOGRAPHIC_FOLD_EVIDENCE +from microcosm.build.uk_runtime.national_frame import UK_NATIONAL_SCHEMA +from microcosm.build.uk_runtime.private_school_attendance import ( + PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE, + PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE, +) +from microcosm.build.uk_runtime.spi_band_donors import ( + HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, + PERSON_IS_SPI_INCOME_BAND_CARRIER, + SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN, + UK_SPI_INCOME_BAND_DONORS_STAGE_NAME, +) +from microcosm.build.uk_runtime.spi_support import ( + SPI_SYNTHETIC_SUPPORT_CHANNEL, + UK_SPI_SUPPORT_STAGE_NAME, + support_channel_column, + support_source_id_column, +) +from microcosm.frame import Frame +from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine +from microcosm.graph import ContentStore, compile_graph, run_graph +from test_support.microcosm_graph.acceptance_h_parity import UK_SPINE_PARITY, _require + +#: The clone count held to the fixture's one clone. +_CLONES = 4 +G1 = GEOGRAPHIC_EXPAND_STAGE_NAME +G1B = GEOGRAPHIC_IDENTITY_STAGE_NAME +SPI = UK_SPI_SUPPORT_STAGE_NAME +DONORS = UK_SPI_INCOME_BAND_DONORS_STAGE_NAME +#: The spine's stage count since the clone-first move: 54 and G1, G1b. +_STAGE_COUNT = 56 +#: The first stage after the clones, and the last before spi_support_channel, +#: which is the first engine call after G1. +_FIRST_DRAW = "frs_take_up" +_LAST_DRAW = "frs_brma" +#: The draws that pool over the clones before spi_support_channel: +#: frs_take_up's anchored residual takes ``int(rate x units)`` of the pool's +#: units, and frs_person_draws' apprentice walk meets a weighted target over +#: the pool, so clone 0's draws need not be one clone's. Every other draw from +#: frs_take_up to frs_brma is keyed on the row's own id, which clone 0 shares +#: with one clone. +_POOLED_DRAWS = { + ("benunit", "would_claim_child_benefit"), + ("benunit", "would_claim_pc"), + ("benunit", "would_claim_uc"), + ("person", "is_apprentice"), +} +#: frs_brma's draw, whose count-table cell reads the engine's LHA_category. +_BRMA = ("household", "brma") +#: The populations the exact checks compare, by stage. +_KEPT = (G1, G1B, _LAST_DRAW, SPI, DONORS) +#: The fits that fold geographic clones into their survey records, each +#: receipting the fold at K > 1 (hmrc_spi_income_spine's two forests share one +#: receipt). +_FOLDING_STAGES = frozenset( + { + "hmrc_spi_income_spine", + "spi_employment_shell", + "spi_housing_shell", + "uc_reporter_redraw", + "uc_capital_coherence", + "salary_sacrifice", + } +) +#: The six support-clip stages and property components, whose receipts carry +#: the clone count their gates scale by only at K > 1. +_CLONE_SCALED_RECEIPTS = frozenset( + { + "was_wealth", + "was_lisa", + "nts_bus_travel", + "lcfs_consumption", + "etb_vat", + "etb_services", + "property_components", + } +) +#: Each stage's exact-total correction, whose inputs the runs record so the +#: rows it moved are where the stage's output weights differ from them. +_CORRECTIONS = { + SPI: (spi_support, "_importance_weights_with_exact_total"), + DONORS: (spi_band_donors, "_exact_incumbent_total"), +} +#: Machine epsilon, the 2.2e-16 of the corrected rows' tolerance. +_EPSILON = float(np.finfo(np.float64).eps) +#: How far a share over every clone may sit from one clone's. The clones draw +#: on their own keyed uniforms, and from spi_income_band_donors on at its +#: common mass scale, so a weighted share over all four differs from one +#: clone's by the draws' sampling error on the fixture's 135 households, which +#: stays under 0.15 on every output here. The bound still catches a stage that +#: leaves three clones of four unset wherever one clone's share passes 0.27, +#: and the presence check catches it at any share. +_SHARE_TOLERANCE = 0.2 +#: Below this many rows a value of one clone's draws can be missing from +#: another clone's own draws by chance, so the presence check reads only +#: outputs that one clone gives at least this many rows. +_PRESENCE_ROWS = 3 +#: The relative tolerance of a float a stage sums in another order at K: a +#: household mass, or a receipt target over the clones' rows. +_SUM_RTOL = 1e-12 + + +@dataclass(frozen=True) +class _Run: + """One clone count's run of the whole spine, as each node left it.""" + + clones: int + order: tuple[str, ...] + stage_names: tuple[str, ...] + hits: Mapping[str, bool] + calls: Mapping[str, tuple[int, ...]] + masses: Mapping[str, float] + problems: Mapping[str, tuple[str, ...]] + shares: Mapping[tuple[str, str], tuple[float, tuple[int, ...]]] + frames: Mapping[str, Frame] + handed: Mapping[str, tuple[np.ndarray, ...]] + evidence: Mapping[str, Mapping[str, object]] + + +def _sources(directory: Path, clones: int) -> Path: + """The H2 fixture's sources, at ``clones`` geographic clones.""" + + source = _require(UK_SPINE_PARITY / "sources", "tools/graph_uk_spine_fixture.py") + if clones == 1: + return source + target = directory / "sources" + shutil.copytree(source, target) + descriptor = json.loads((target / "fixture.json").read_text(encoding="utf-8")) + assert descriptor["config"]["geographic_clones"] == 1 + descriptor["config"]["geographic_clones"] = clones + (target / "fixture.json").write_text( + json.dumps(descriptor, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return target + + +def _output_node(nodes: set[str], stage: str) -> str: + """The node whose population holds ``stage``'s outputs.""" + + if stage == "frs_spine": + return "create_uk_frs" + return f"{stage}.owned" if f"{stage}.owned" in nodes else stage + + +def _entity_weights(frame: Frame, entity: str) -> np.ndarray: + """Each row's household weight: a person's household's, a benefit unit's + members' household's.""" + + household = frame.table("household") + weights = pd.Series( + np.asarray(frame.weights_for("household").values, dtype=float), + index=household["household_id"].to_numpy(), + ) + if entity == "household": + return weights.to_numpy() + person = frame.table("person") + homes = person["person_household_id"] + if entity == "person": + return weights.loc[homes.to_numpy()].to_numpy() + unit_homes = homes.groupby(person["person_benunit_id"].to_numpy()).first() + units = frame.table("benunit")["benunit_id"].to_numpy() + return weights.loc[unit_homes.loc[units].to_numpy()].to_numpy() + + +def _nonzero(values: pd.Series) -> np.ndarray: + if pd.api.types.is_bool_dtype(values.dtype): + return values.fillna(False).to_numpy(dtype=bool) + if pd.api.types.is_numeric_dtype(values.dtype): + numbers = values.to_numpy(dtype=float, na_value=np.nan) + return (numbers != 0.0) & ~np.isnan(numbers) + return values.fillna("").astype(str).ne("").to_numpy() + + +def _clone_index_problems(frame: Frame, clones: int) -> tuple[str, ...]: + """Where ``frame`` breaks the clone-index invariant: a table without its + index, an index of another dtype or other values than exactly 0 to + K - 1, and members whose index is not their household's and benefit + unit's.""" + + problems = [] + for entity in UK_NATIONAL_SCHEMA.entities: + column = ladder_clone_index_column(entity) + table = frame.table(entity) + if column not in table.columns: + problems.append(f"the {entity} table lacks {column!r}") + continue + if table[column].dtype != np.dtype("int64"): + problems.append(f"{column!r} is not int64") + if sorted(pd.unique(table[column]).tolist()) != list(range(clones)): + problems.append(f"{column!r} does not take exactly the values 0 to K - 1") + if problems: + return tuple(problems) + person = frame.table("person") + own = person[ladder_clone_index_column("person")].to_numpy() + for group in UK_NATIONAL_SCHEMA.group_entities: + units = frame.table(group).set_index(UK_NATIONAL_SCHEMA.id_column(group)) + theirs = person[UK_NATIONAL_SCHEMA.membership_column(group)].map( + units[ladder_clone_index_column(group)] + ) + off = int((theirs.to_numpy() != own).sum()) + if off: + problems.append( + f"{off} person row(s) carry another index than their {group}" + ) + return tuple(problems) + + +def _run(directory: Path, clones: int, pending: Mapping[str, list]) -> _Run: + source = _sources(directory, clones) + stages, implementations = fixture_stage_plan_inputs(source) + stage_names = tuple(stage.stage for stage in stages) + by_name = {stage.stage: stage for stage in stages} + graph = uk_spine_graph(geographic_clones=clones) + compiled = compile_graph(graph) + order = tuple(compiled.order) + nodes = set(order) + first_clone_node = order.index(G1) + outputs = { + _output_node(nodes, stage): stage + for stage in stage_names[stage_names.index(_FIRST_DRAW) :] + } + kept = {_output_node(nodes, stage): stage for stage in _KEPT} + calls: dict[str, tuple[int, ...]] = {} + masses: dict[str, float] = {} + problems: dict[str, tuple[str, ...]] = {} + shares: dict[tuple[str, str], tuple[float, tuple[int, ...]]] = {} + frames: dict[str, Frame] = {} + handed: dict[str, tuple[np.ndarray, ...]] = {} + + def observe(node_id: str, population) -> None: + frame = population.frame + calls[node_id] = tuple(pending["calls"]) + pending["calls"].clear() + for stage in _CORRECTIONS: + if pending[stage]: + handed[node_id] = tuple(pending[stage]) + pending[stage].clear() + masses[node_id] = math.fsum(frame.weights_for("household").values) + if order.index(node_id) >= first_clone_node: + problems[node_id] = _clone_index_problems(frame, clones) + else: + problems[node_id] = tuple( + f"the {entity} table carries its clone index before {G1}" + for entity in frame.entities + if ladder_clone_index_column(entity) in frame.table(entity).columns + ) + if node_id in outputs: + stage = by_name[outputs[node_id]] + for column in (*stage.outputs, *stage.rewrites): + holders = [ + entity + for entity in frame.entities + if column in frame.table(entity).columns + ] + if not holders: + # Recorded so the share test names it; no stage drops + # a column it declares. + shares[(stage.stage, column)] = (math.nan, ()) + continue + (entity,) = holders + table = frame.table(entity) + present = _nonzero(table[column]) + weights = _entity_weights(frame, entity) + index = table[ladder_clone_index_column(entity)].to_numpy() + shares[(stage.stage, column)] = ( + float(weights[present].sum() / weights.sum()), + tuple( + int(present[index == clone].sum()) for clone in range(clones) + ), + ) + if node_id in kept: + frames[kept[node_id]] = frame + + store = ContentStore(directory / "store") + manifest = run_graph( + compiled, + sources={"frs": source}, + store=store, + # Built against the graph the run is handed, as the spine build does. + kernels=uk_registry(dict(implementations), graph=graph), + resume="forbid", + decisions=(), + retain_populations=(compiled.versions[order[-1]],), + _population_observer=observe, + ) + artifacts = load_spine_stage_artifacts(manifest, store, stage_names=stage_names) + return _Run( + clones=clones, + order=order, + stage_names=stage_names, + hits={node: receipt.hit for node, receipt in manifest.nodes.items()}, + calls=calls, + masses=masses, + problems=problems, + shares=shares, + frames=frames, + handed=handed, + evidence={stage: document["evidence"] for stage, document in artifacts.items()}, + ) + + +@pytest.fixture(scope="module") +def runs(tmp_path_factory) -> dict[int, _Run]: + """The H2 fixture's whole spine at one clone and at four, each engine + call's simulations and each exact-total correction's inputs recorded.""" + + pending: dict[str, list] = {"calls": [], **{stage: [] for stage in _CORRECTIONS}} + materialize = PolicyEngineUKEngine.materialize + + def counted(self, bundle, variables, period, **kwargs): + pending["calls"].append(self.simulation_batches(bundle)) + return materialize(self, bundle, variables, period, **kwargs) + + def recording(function, stage: str): + def recorded(values, *args, **kwargs): + pending[stage].append(np.array(values, dtype=float, copy=True)) + return function(values, *args, **kwargs) + + return recorded + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(PolicyEngineUKEngine, "materialize", counted) + for stage, (module, name) in _CORRECTIONS.items(): + patch.setattr(module, name, recording(getattr(module, name), stage)) + return { + clones: _run(tmp_path_factory.mktemp(f"k{clones}"), clones, pending) + for clones in (1, _CLONES) + } + + +# --- helpers ------------------------------------------------------------------------- + + +def _weights(frame: Frame) -> pd.Series: + return pd.Series( + np.asarray(frame.weights_for("household").values, dtype=float), + index=frame.table("household")["household_id"].to_numpy(), + ) + + +def _clone_zero(frame: Frame, entity: str) -> pd.DataFrame: + table = frame.table(entity) + return table.loc[table[ladder_clone_index_column(entity)].eq(0)] + + +def _clone_zero_ids(frame: Frame) -> set[int]: + return set(_clone_zero(frame, "household")["household_id"].tolist()) + + +def _copied(frame: Frame, clone: int) -> set[int]: + """The households spi_support_channel copied in ``clone``, by the copies' + source ids.""" + + household = frame.table("household") + rows = household[support_channel_column("household")].eq( + SPI_SYNTHETIC_SUPPORT_CHANNEL + ) & household[ladder_clone_index_column("household")].eq(clone) + return set(household.loc[rows, support_source_id_column("household")].tolist()) + + +def _seating(frame: Frame, clone: int) -> dict[int, float]: + """The band donors' seats in ``clone``: each donor's source household, and + its band.""" + + household = frame.table("household") + donors = household.loc[ + household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR] + & household[ladder_clone_index_column("household")].eq(clone) + ] + return dict( + zip( + donors[support_source_id_column("household")].tolist(), + donors[SPI_INCOME_BAND_DONOR_LOWER_BOUND_COLUMN].tolist(), + strict=True, + ) + ) + + +def _carriers(frame: Frame, clone: int) -> set[int]: + """The band donors' carriers in ``clone``, by the person each copy was made + from.""" + + person = frame.table("person") + carriers = person.loc[ + person[PERSON_IS_SPI_INCOME_BAND_CARRIER] + & person[ladder_clone_index_column("person")].eq(clone) + ] + return set(carriers[support_source_id_column("person")].tolist()) + + +def _copies_off_their_parents_clone(frame: Frame) -> int: + """How many households carry another clone index than the household + their support lineage names, which spi_support_channel and + spi_income_band_donors keep beside their copies (a household that is no + copy names itself).""" + + household = frame.table("household") + index = ladder_clone_index_column("household") + by_id = household.set_index("household_id")[index] + parents = household[support_source_id_column("household")].map(by_id) + return int((parents.isna() | (parents.to_numpy() != household[index])).sum()) + + +def _moved(run: _Run, stage: str) -> frozenset[int]: + """The households ``stage``'s exact-total correction moved: where its + output weights differ from the weights it was handed.""" + + (values,) = run.handed[stage] + weights = _weights(run.frames[stage]) + return frozenset(weights.index[weights.to_numpy() != values].tolist()) + + +def _corrected(one: _Run, many: _Run, stage: str) -> set[int]: + """The households an exact-total correction moved, by spi_support_channel + or ``stage``, at either clone count.""" + + return { + household_id + for run in (one, many) + for earlier in (SPI, stage) + if earlier in run.handed + for household_id in _moved(run, earlier) + } + + +def _nesting_failures(one: _Run, many: _Run, stage: str) -> tuple[Counter, Counter]: + """K x each clone-0 weight against one clone's, at ``stage``'s version, in + two classes at a power-of-two K: rows an exact-total correction moved + (``_corrected``) within ``rows x K x 2.2e-16`` relative, every other row + bitwise. Returns the classes' counts and their failures'.""" + + reference = _weights(one.frames[stage]) + clone_zero = _clone_zero_ids(many.frames[stage]) + assert clone_zero == set(reference.index), ( + f"Clone 0 at K={many.clones} after {stage} is not the one-clone " + "frame's households." + ) + scaled = _weights(many.frames[stage]).loc[reference.index] + corrected = _corrected(one, many, stage) + bound = Fraction(len(reference) * many.clones) * Fraction(_EPSILON) + classes: Counter[str] = Counter() + failures: Counter[str] = Counter() + for household_id, weight, expected in zip( + reference.index.tolist(), scaled.tolist(), reference.tolist(), strict=True + ): + difference = abs(Fraction(weight) * many.clones - Fraction(expected)) + if household_id in corrected: + classes["corrected"] += 1 + if difference > bound * Fraction(expected): + failures["corrected rows beyond rows x K x 2.2e-16 relative"] += 1 + else: + classes["bitwise"] += 1 + if difference: + failures["uncorrected rows not bitwise at a power-of-two K"] += 1 + return classes, failures + + +# --- every stage, every engine call --------------------------------------------------- + + +def test_every_stage_runs_and_every_engine_call_after_g1_runs_clone_by_clone( + runs, +) -> None: + one, many = runs[1], runs[_CLONES] + assert one.stage_names == many.stage_names + assert len(many.stage_names) == _STAGE_COUNT + assert one.order == many.order + for run in (one, many): + assert set(run.hits) == set(run.order) == set(run.calls) + assert not any(run.hits.values()), f"A node hit its record at K={run.clones}." + first = many.order.index(G1) + called = [node for node in many.order if many.calls[node]] + after = [node for node in called if many.order.index(node) >= first] + assert {"frs_brma", "private_school_attendance"} <= set(after) + for node in called: + # As many calls as at one clone: no stage calls the engine per clone. + assert len(many.calls[node]) == len(one.calls[node]), node + assert set(one.calls[node]) == {1}, node + expected = _CLONES if many.order.index(node) >= first else 1 + assert set(many.calls[node]) == {expected}, (node, expected) + assert set(called) == {node for node in one.order if one.calls[node]} + # private_school_attendance restates attendance over the pool when its + # call runs in batches, and keeps the engine's own at one clone. + routes = { + run.clones: run.evidence["private_school_attendance"]["rank_route"] + for run in (one, many) + } + assert routes == { + 1: PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE, + _CLONES: PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE, + } + + +def test_every_population_keeps_the_clone_index_invariant(runs) -> None: + for run in runs.values(): + broken = {node: found for node, found in run.problems.items() if found} + assert not broken, f"At K={run.clones}: {broken}." + for stage in (SPI, DONORS): + off = _copies_off_their_parents_clone(run.frames[stage]) + assert off == 0, ( + f"{off} household(s) after {stage} at K={run.clones} carry another " + "clone index than the household their support lineage names." + ) + + +def test_every_node_keeps_the_one_clone_household_mass(runs) -> None: + """The clones split each household's weight, and every later stage + conserves or reallocates mass as at one clone: each node's household + mass is K = 1's, summed in another order.""" + + one, many = runs[1], runs[_CLONES] + off = [ + node + for node in many.order + if not math.isclose(many.masses[node], one.masses[node], rel_tol=_SUM_RTOL) + ] + assert not off, f"{len(off)} node(s) at K={_CLONES} change the household mass." + + +# --- exact ------------------------------------------------------------------------------ + + +@pytest.mark.parametrize("stage", (G1, G1B)) +def test_clone_zero_after_the_clones_is_the_one_clone_frame(runs, stage: str) -> None: + """Clone-major: clone 0 is the frame's first rows, every column and dtype + as at one clone (the ids, G1's columns and G1b's key among them), at K + times its weight bit for bit, since 1/4 splits exactly.""" + + one, many = runs[1].frames[stage], runs[_CLONES].frames[stage] + written = {*UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, GEOGRAPHY_HOUSEHOLD_KEY_COLUMN} + for entity in UK_NATIONAL_SCHEMA.entities: + clone_zero = _clone_zero(many, entity) + assert clone_zero.index.tolist() == list(range(len(one.table(entity)))) + pd.testing.assert_frame_equal(clone_zero, one.table(entity), check_exact=True) + written -= set(one.table(entity).columns) + if stage == G1: + written.discard(GEOGRAPHY_HOUSEHOLD_KEY_COLUMN) + assert not written, f"Some of {stage}'s columns were not compared." + weights = _weights(many).loc[_weights(one).index].to_numpy() + assert (weights * _CLONES).tobytes() == _weights(one).to_numpy().tobytes() + + +def _households_off_one_clone_on_the_pooled_draws(one: Frame, many: Frame) -> set[int]: + """The households where a clone-0 pooled draw (``_POOLED_DRAWS``) is not + one clone's: a benefit unit's take-up flag or a member's apprentice flag. + frs_brma takes each unit's count-table cell from the engine's + LHA_category, which reads the household's members and units (a claim to + Universal Credit decides whose non-dependants count), so only in these + households may clone 0's BRMA lie in another cell.""" + + person = one.table("person") + homes = { + "person": person.set_index("person_id")["person_household_id"], + "benunit": person.groupby("person_benunit_id")["person_household_id"].first(), + } + ids = {"person": "person_id", "benunit": "benunit_id"} + off: set[int] = set() + for entity, column in sorted(_POOLED_DRAWS): + left = one.table(entity).set_index(ids[entity])[column] + right = ( + _clone_zero(many, entity).set_index(ids[entity])[column].reindex(left.index) + ) + moved = left.index[left.to_numpy() != right.to_numpy()] + off |= set(homes[entity].loc[moved].tolist()) + return off + + +def test_clone_zero_keeps_the_keyed_per_row_draws_through_frs_brma(runs) -> None: + """Through frs_brma clone 0 is the one-clone frame on every column but + the draws that pool over the clones (``_POOLED_DRAWS``) and the BRMA of + the households where those differ: every other draw from frs_take_up to + frs_brma is keyed on the row's own id, and the BRMA draw's cell reads the + LHA_category the batched engine computes for clone 0 from that clone's + rows alone, so it is one clone's wherever the household's inputs are. + The other households' BRMA is left to the share class.""" + + one, many = runs[1].frames[_LAST_DRAW], runs[_CLONES].frames[_LAST_DRAW] + compared: set[tuple[str, str]] = set() + differing: list[str] = [] + for entity in UK_NATIONAL_SCHEMA.entities: + reference = one.table(entity).reset_index(drop=True) + clone_zero = _clone_zero(many, entity).reset_index(drop=True) + assert list(clone_zero.columns) == list(reference.columns), entity + for column in reference.columns: + if (entity, column) in _POOLED_DRAWS or (entity, column) == _BRMA: + continue + compared.add((entity, column)) + if not clone_zero[column].equals(reference[column]): + differing.append(f"{entity}.{column}") + assert not differing, f"Clone 0 after {_LAST_DRAW} differs on {differing}." + moved = _households_off_one_clone_on_the_pooled_draws(one, many) + entity, column = _BRMA + reference = one.table(entity).set_index("household_id")[column] + clone_zero = ( + _clone_zero(many, entity) + .set_index("household_id")[column] + .reindex(reference.index) + ) + kept = ~reference.index.isin(list(moved)) + assert kept.any(), "The pooled draws moved every household; no BRMA compared." + off = int((clone_zero.to_numpy()[kept] != reference.to_numpy()[kept]).sum()) + assert off == 0, ( + f"{off} household(s) whose pooled draws are one clone's draw another " + f"BRMA in clone 0 ({len(moved)} household(s) left to the share class)." + ) + compared.add(_BRMA) + stages = runs[1].stage_names + keyed = { + (entity, column) + for stage in ("frs_person_draws", "frs_household_draws", _LAST_DRAW) + for column in _stage_outputs(stages, stage) + for entity in UK_NATIONAL_SCHEMA.entities + if column in one.table(entity).columns + } - _POOLED_DRAWS + assert keyed and keyed <= compared + + +def _stage_outputs(stage_names: tuple[str, ...], stage: str) -> tuple[str, ...]: + from microcosm.build.country_spec import load_country_spec + + assert stage in stage_names + declared = load_country_spec("uk").sources.stage_map()[stage] + return (*declared.outputs, *declared.rewrites) + + +def test_the_support_channel_copies_the_one_clone_households_in_clone_zero( + runs, +) -> None: + one, many = runs[1], runs[_CLONES] + reference = _copied(one.frames[SPI], 0) + assert reference + assert _copied(many.frames[SPI], 0) == reference + assert any( + _copied(many.frames[SPI], clone) != reference for clone in range(1, _CLONES) + ) + classes, failures = _nesting_failures(one, many, SPI) + assert not failures, f"After {SPI} at K={_CLONES} ({dict(classes)}): {failures}." + + +def test_the_band_donors_seat_the_one_clone_households_in_clone_zero(runs) -> None: + one, many = runs[1].frames[DONORS], runs[_CLONES].frames[DONORS] + # Clone 0 holds exactly the one-clone frame's households, the support + # channel's copies and the donors among them. + assert _clone_zero_ids(many) == set(_weights(one).index) + seating = _seating(one, 0) + assert seating + assert _seating(many, 0) == seating + assert _carriers(many, 0) == _carriers(one, 0) + seats = Counter(seating.values()) + for clone in range(_CLONES): + assert Counter(_seating(many, clone).values()) == seats, clone + + +# --- the band donors' weights --------------------------------------------------------- + + +def _mass_scales(runs) -> tuple[float, float, str]: + one = float(runs[1].evidence[DONORS]["mass_scale"]) + many = float(runs[_CLONES].evidence[DONORS]["mass_scale"]) + report = ( + f"{DONORS}'s mass scale at K={_CLONES}: equal to K=1's {many == one}, " + f"below it {many < one}" + ) + return one, many, report + + +def test_the_band_donors_report_their_mass_scale_and_scale_clone_zero_donors_by_it( + runs, +) -> None: + """Reported first: spi_income_band_donors' mass scale at one clone and at + four. Every clone's donors take one scale, the smallest any clone's region + allows, so each clone-0 donor weighs one clone's donor weight times the + ratio of the two scales, to the rounding of the two products (a few + ulps).""" + + scale_one, scale_many, report = _mass_scales(runs) + one, many = runs[1].frames[DONORS], runs[_CLONES].frames[DONORS] + household = one.table("household").set_index("household_id") + donors = household.index[household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR]] + assert len(donors), report + left = _CLONES * _weights(many).loc[donors].to_numpy() * scale_one + right = _weights(one).loc[donors].to_numpy() * scale_many + off = int((np.abs(left - right) > 4 * np.spacing(right)).sum()) + assert off == 0, f"{off} clone-0 donor(s) off the scales' ratio; {report}." + + +def test_the_band_donors_fund_each_clone_zero_stratum_at_one_ratio(runs) -> None: + """K x clone-0 weights after spi_income_band_donors, over one clone's. + The stage scales every incumbent of a funding stratum by the stratum's + one factor, so in each stratum of clone 0 every incumbent takes one + ratio, the stratum's factor at K = 4 over its factor at one clone, to the + rounding of the two products (a few ulps), apart from the rows an + exact-total correction moved; a stratum without donors takes exactly + one, both its factors being one; and where the two mass scales are + equal, every ratio is one, so K x clone-0 weights nest. On the H2 + fixture's capped rung the scale at K = 4 is below one clone's, so the + strata with donors take ratios other than one. Whether the stage takes a + mass scale per clone is open (microcosm#931).""" + + one, many = runs[1], runs[_CLONES] + scale_one, scale_many, report = _mass_scales(runs) + reference = _weights(one.frames[DONORS]) + ratio = ( + _CLONES * _weights(many.frames[DONORS]).loc[reference.index] / reference + ).to_numpy() + household = many.frames[DONORS].table("household") + by_id = household["household_id"].to_numpy() + strata = ( + pd.Series(spi_band_donors._funding_strata(household).to_numpy(), index=by_id) + .loc[reference.index] + .to_numpy() + ) + donor = ( + pd.Series( + household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR].to_numpy(dtype=bool), + index=by_id, + ) + .loc[reference.index] + .to_numpy() + ) + corrected = reference.index.isin(list(_corrected(one, many, DONORS))) + funded = set(strata[donor]) + ratios: dict[str, float] = {} + off: Counter[str] = Counter() + for stratum in sorted(set(strata[~donor])): + values = ratio[(strata == stratum) & ~donor & ~corrected] + if not values.size: + continue + ratios[stratum] = float(values[0]) + if (np.abs(values - values[0]) > 8 * np.spacing(values[0])).any(): + off["strata whose incumbents take more than one ratio"] += 1 + if stratum not in funded and not (values == 1.0).all(): + off["strata without donors off ratio one"] += 1 + assert funded and funded <= set(ratios), report + if scale_many == scale_one and any( + abs(value - 1.0) > 8 * np.spacing(1.0) for value in ratios.values() + ): + off["strata off ratio one at equal mass scales"] += 1 + assert not off, f"After {DONORS} at K={_CLONES}: {dict(off)}; {report}." + + +# --- tolerance classes --------------------------------------------------------------- + + +def test_the_pooled_draws_keep_their_k_invariant_receipt_fields(runs) -> None: + """frs_take_up, frs_person_draws and frs_household_draws count each + flag's population in true sources, and frs_person_draws' apprentice walk + its target, masses and supports, so these are one clone's at K = 4; each + target and mass, summed over the clones' rows, is one clone's to its + summation order. The realized shares are left to the share class.""" + + one, many = runs[1].evidence, runs[_CLONES].evidence + for stage in (_FIRST_DRAW, "frs_person_draws", "frs_household_draws"): + flags = { + run: { + row["output"]: row for row in evidence[stage]["take_up_signal"]["flags"] + } + for run, evidence in (("one", one), ("many", many)) + } + assert set(flags["one"]) == set(flags["many"]), stage + for output, row in flags["one"].items(): + other = flags["many"][output] + for field in ("population_support", "distinct_values", "rate_key"): + assert other.get(field) == row.get(field), (stage, output, field) + if "target" in row: + assert math.isclose(other["target"], row["target"], rel_tol=_SUM_RTOL) + apprentices = { + run: evidence["frs_person_draws"]["apprentice_receipt"] + for run, evidence in (("one", one), ("many", many)) + } + for field in ( + "target", + "rate", + "population_mass", + "eligible_mass", + "population_support", + "eligible_support", + "cap_bound", + ): + expected, found = apprentices["one"][field], apprentices["many"][field] + if isinstance(expected, float) and isinstance(found, float): + assert math.isclose(found, expected, rel_tol=_SUM_RTOL), field + else: + assert found == expected, field + + +def test_every_later_output_is_carried_by_every_clone_at_a_nearby_share( + runs, +) -> None: + """From frs_take_up on, every stage's declared outputs (frs_take_up's + residual, frs_person_draws' walk, cgt_support_split and everything + downstream of an engine call among them) at K = 4: a weighted share over + the clones within + ``_SHARE_TOLERANCE`` of one clone's, and on every clone wherever one + clone gives it ``_PRESENCE_ROWS`` rows or more.""" + + one, many = runs[1].shares, runs[_CLONES].shares + assert set(one) == set(many) + assert {stage for stage, _ in one} >= { + _FIRST_DRAW, + "frs_person_draws", + "cgt_support_split", + "private_school_attendance", + } + apart = sorted( + f"{stage}.{column}" + for (stage, column), (share, _) in one.items() + if abs(many[(stage, column)][0] - share) > _SHARE_TOLERANCE + ) + assert not apart, f"Outputs whose share at K={_CLONES} is apart: {apart}." + absent = sorted( + f"{stage}.{column}" + for (stage, column), (_, (rows,)) in one.items() + if rows >= _PRESENCE_ROWS and min(many[(stage, column)][1]) == 0 + ) + assert not absent, f"Outputs a clone at K={_CLONES} does not carry: {absent}." + + +# --- receipts --------------------------------------------------------------------------- + + +def _found(value: object, key: str) -> list[object]: + """Every value stored under ``key`` anywhere in ``value``.""" + + found: list[object] = [] + if isinstance(value, Mapping): + for name, item in value.items(): + if name == key: + found.append(item) + found.extend(_found(item, key)) + elif isinstance(value, list | tuple): + for item in value: + found.extend(_found(item, key)) + return found + + +def test_the_receipts_carry_the_clone_count_and_the_fold_receipts(runs) -> None: + one, many = runs[1].evidence, runs[_CLONES].evidence + for stage in (G1, SPI, DONORS): + assert one[stage]["clone_count"] == 1, stage + assert many[stage]["clone_count"] == _CLONES, stage + # The counts in true sources are one clone's. + assert many[G1]["source_households"] == one[G1]["source_households"] + for field in ("expected_spi_households", "source_households"): + assert many[SPI][field] == one[SPI][field], field + for field in ("eligible_households", "seating_scale"): + assert many[DONORS][field] == one[DONORS][field], field + for evidence in (one, many): + assert evidence[DONORS]["funding_within_share"] is True + assert evidence[DONORS]["funding_matches_bands"] is True + counted = { + clones: { + stage + for stage, document in evidence.items() + if document is not None and _found(document, "clone_count") + } + for clones, evidence in ((1, one), (_CLONES, many)) + } + assert counted[_CLONES] - counted[1] >= _CLONE_SCALED_RECEIPTS | _FOLDING_STAGES + assert not counted[1] & (_CLONE_SCALED_RECEIPTS | _FOLDING_STAGES) + for stage in counted[_CLONES]: + assert set(_found(many[stage], "clone_count")) == {_CLONES}, stage + folds = { + clones: { + stage: _found(document, GEOGRAPHIC_FOLD_EVIDENCE) + for stage, document in evidence.items() + if document is not None and _found(document, GEOGRAPHIC_FOLD_EVIDENCE) + } + for clones, evidence in ((1, one), (_CLONES, many)) + } + assert folds[1] == {} + assert set(folds[_CLONES]) == _FOLDING_STAGES + for stage, receipts in folds[_CLONES].items(): + for receipt in receipts: + assert receipt["clone_count"] == _CLONES, stage + assert {"records", "varied_records", "varied_share"} <= set(receipt), stage diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index bb1fe83deb..d08e4b4da9 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -3,12 +3,15 @@ Recomputes every E4 column twice from the pure derivations — once in the frame's row order, once on row-permuted tables — un-permutes by entity id, and also compares the original-order recomputation against the columns -stored in the artifact. Exit status is nonzero on any mismatch. +stored in the artifact. Exit status is nonzero on any mismatch. With the +spine's build sidecar (``--spine-sidecar``) the frame's geographic clone +count is first checked against the spine's (microcosm#931). """ from __future__ import annotations import argparse +import hashlib import json import sys from collections.abc import Callable, Mapping, Sequence @@ -39,6 +42,7 @@ uc_age_eligible_benunits, uk_take_up_population_policy, ) +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE from microcosm.build.uk_runtime.national_frame import ( load_uk_national_frame, @@ -53,6 +57,7 @@ from microcosm.build.uk_runtime.spi_band_donors import ( HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, SPI_INCOME_BAND_DONOR_CLONE_INDEX, + UK_SPI_INCOME_BAND_DONORS_STAGE_NAME, ) from microcosm.build.uk_runtime.was_wealth import ( allocate_student_loan_balance_to_people, @@ -99,30 +104,54 @@ def e4_identity_receipt( *, contract, count_resource: Mapping[str, object], - lha_category: Sequence[object], + lha_category: Sequence[object] | None = None, + engine=None, permutation_seed: int, population_policy=None, + rewritten_later: Mapping[str, Sequence[str]] | None = None, + reversed_stages: Sequence[str] | None = None, ) -> dict[str, object]: """Recompute every E4 column in original and permuted row order. - ``population_policy`` is the engine's working-age bounds the take-up draw - used (``uk_take_up_population_policy``); it is read from the engine when - not supplied, so a hermetic caller passes one. + ``frame`` is the population E4's four stages drew on, at the weights they + read: every geographic clone of the raw FRS rows, before the SPI support + channel stacked its copies (``_frame_as_draw_stages_saw``, which the tool + passes). ``population_policy`` is the engine's working-age bounds the + take-up draw used (``uk_take_up_population_policy``); it is read from the + engine when not supplied, so a hermetic caller passes one. + + ``lha_category`` is the BRMA draw's predictor, aligned to the benefit-unit + rows. Without it, ``engine`` materializes it on the frame the BRMA stage + read, as far as the artifact restores it (``_brma_stage_lha_category``): + LHA_category reads would_claim_uc, which later stages rewrite, so the + final frame's flags would draw some benefit units from another cell. Two claims are receipted: a row permutation of the input tables changes no assignment per entity id, and the original-order recomputation equals - the columns stored in the artifact (re-derivation identity). + the columns stored in the artifact (re-derivation identity). A column a + stage after its own declares writing, in the packaged manifest + (``rewritten_later``, by default ``_later_writers`` over the E4 stages, + the stages in ``reversed_stages`` excepted), may hold the later stage's + value, so it is not compared: the receipt names the stages that wrote + it and, under ``rewritten_later_still_equal``, the columns the store + still holds bitwise (a later stage may write only rows the scope drops, + such as copies). ``is_apprentice`` is held exactly, or where it differs, + to its walk on the restored weights (``_apprentice_walk_holds``), since + those are the stage's to rounding only; the receipt names it then. """ + from microcosm.build.uk_runtime.frs_person_draws import ( + APPRENTICE_OUTPUT, + APPRENTICE_RATE_KEY, + ) + person = frame.table("person") benunit = frame.table("benunit").copy() household = frame.table("household") if population_policy is None: population_policy = uk_take_up_population_policy(uk_time_period(frame)) - if len(lha_category) != len(benunit): - raise ValueError("LHA_category materialization must align to benunit rows.") - benunit["LHA_category"] = [_enum_name(value) for value in lha_category] - benunit["region"] = _benunit_regions(person, household, benunit) + if reversed_stages is None: + reversed_stages = _SCOPE_REVERSALS # is_apprentice's rate among eligible employees reads the household # weights, which a row permutation carries with each household id. weight_by_household = dict( @@ -133,7 +162,9 @@ def e4_identity_receipt( ) ) - def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: + def draws(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: + """The take-up, person and household draws, keyed by entity id.""" + anchors = aggregate_person_reported_to_benunit(person_t, benunit_t) take_up = derive_frs_take_up( benunit_t, @@ -165,6 +196,32 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: person_draws.index = person_t["person_id"].to_numpy() household_draws = derive_frs_household_draws(household_t, contract=contract) household_draws.index = household_t["household_id"].to_numpy() + return { + "benunit": take_up, + "person": person_draws, + "household": household_draws, + } + + lha_scope = None + if lha_category is None: + if engine is None: + raise ValueError( + "e4 identity receipt: pass lha_category, or an engine to " + "materialize it on the frame the BRMA stage read." + ) + lha_category, lha_scope = _brma_stage_lha_category( + frame, + engine=engine, + drawn=draws(person, benunit, household), + reversed_stages=reversed_stages, + ) + if len(lha_category) != len(benunit): + raise ValueError("LHA_category materialization must align to benunit rows.") + benunit["LHA_category"] = [_enum_name(value) for value in lha_category] + benunit["region"] = _benunit_regions(person, household, benunit) + + def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: + values = draws(person_t, benunit_t, household_t) seed = UK_BRMA_DECLARED_SEEDS["brma"] benunit_brma = pd.DataFrame( { @@ -174,14 +231,10 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: ), } ) - household_draws["brma"] = collapse_benunit_brma_to_household( + values["household"]["brma"] = collapse_benunit_brma_to_household( person_t, benunit_brma, household_t, seed=seed ) - return { - "benunit": take_up, - "person": person_draws, - "household": household_draws, - } + return values original = recompute(person, benunit, household) rng = np.random.default_rng(permutation_seed) @@ -190,6 +243,12 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: benunit.iloc[rng.permutation(len(benunit))].reset_index(drop=True), household.iloc[rng.permutation(len(household))].reset_index(drop=True), ) + if rewritten_later is None: + rewritten_later = _later_writers( + [column for values in original.values() for column in values.columns], + checked_stages=_E4_STAGES, + reversed_stages=reversed_stages, + ) stored = { "person": frame.table("person").set_index("person_id"), @@ -199,6 +258,9 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: permutation_mismatches: dict[str, list[str]] = {} stored_mismatches: dict[str, list[str]] = {} stored_columns_missing: dict[str, list[str]] = {} + rewritten: dict[str, dict[str, list[str]]] = {} + rewritten_still_equal: dict[str, list[str]] = {} + held_to_tolerance: dict[str, list[str]] = {} for entity, values in original.items(): for column in values.columns: left = values[column] @@ -209,11 +271,34 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: stored_columns_missing.setdefault(entity, []).append(column) continue kept = stored[entity][column].reindex(left.index) - if not np.array_equal( + if column in rewritten_later: + # Not compared, but said where the store still holds the + # recompute: a later writer whose rows the scope does not keep + # (copies only, say) leaves these rows as the stage wrote them. + rewritten.setdefault(entity, {})[column] = list(rewritten_later[column]) + if _equal_where_comparable(left, kept): + rewritten_still_equal.setdefault(entity, []).append(column) + continue + if np.array_equal( left.to_numpy(), kept.to_numpy().astype(left.to_numpy().dtype) ): - stored_mismatches.setdefault(entity, []).append(column) - return { + continue + if (entity, column) == ("person", APPRENTICE_OUTPUT) and ( + _apprentice_walk_holds( + person, + stored["person"][column] + .reindex(person["person_id"].to_numpy()) + .to_numpy(dtype=bool), + person_weights=person["person_household_id"] + .map(weight_by_household) + .to_numpy(dtype=float), + rate=contract.rate(APPRENTICE_RATE_KEY, contract.build_year), + ) + ): + held_to_tolerance.setdefault(entity, []).append(column) + continue + stored_mismatches.setdefault(entity, []).append(column) + receipt: dict[str, object] = { "check": "uk_e4_identity_stability", "permutation_seed": permutation_seed, "identical_under_permutation": not permutation_mismatches, @@ -221,6 +306,25 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: "permutation_mismatches": permutation_mismatches, "stored_mismatches": stored_mismatches, "stored_columns_missing": stored_columns_missing, + "rewritten_later": rewritten, + "rewritten_later_still_equal": rewritten_still_equal, + "held_to_tolerance": held_to_tolerance, + "tolerance_policy": ( + "permutation: bitwise for every column; stored columns: bitwise, " + "except a column a stage after its own declares writing, which is " + "not compared (rewritten_later names the stages, and " + "rewritten_later_still_equal the columns the store still holds " + "bitwise), and is_apprentice, which where it differs is held to " + "its walk on the " + "restored stage-time weights (they are the stage's to rounding): " + "the stored flags must be exactly the eligible employees whose " + "keyed uniform lies at or below the largest flagged one's, and " + "their weight must lie as near the target as the recompute's, to " + f"a relative {_APPRENTICE_WALK_RELATIVE_TOLERANCE:g} of the weight " + "of the people aged 21 to 24 (held_to_tolerance names it then); " + "brma: bitwise, on LHA_category materialized on the frame frs_brma " + "read as far as the artifact restores it (lha_category_scope)" + ), "columns_by_entity": { entity: list(values.columns) for entity, values in original.items() }, @@ -228,6 +332,171 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: entity: int(len(frame.table(entity))) for entity in frame.entities }, } + if lha_scope is not None: + receipt["lha_category_scope"] = lha_scope + return receipt + + +def _equal_where_comparable(recomputed: pd.Series, stored: pd.Series) -> bool: + """Whether ``stored`` holds ``recomputed`` bitwise, False where the + stored column cannot be read as the recompute's dtype: a later stage's + writes need not keep it.""" + + try: + kept = stored.to_numpy().astype(recomputed.to_numpy().dtype) + except (TypeError, ValueError): + return False + return bool(np.array_equal(recomputed.to_numpy(), kept)) + + +def _brma_stage_lha_category( + frame, + *, + engine, + drawn: Mapping[str, pd.DataFrame], + reversed_stages: Sequence[str], +) -> tuple[object, dict[str, object]]: + """LHA_category materialized on the frame the BRMA stage read. + + The stage materialized it on its own input (``UKFRSBRMAStageTransform``), + which carried no column a later stage first writes and the flags E4's + earlier stages drew. policyengine-uk's LHA_category reads both: its + bedrooms and its shared-accommodation test read which family counts the + household's non-dependants (``uc_non_dependants_counted``), which reads + ``claims_universal_credit``, would_claim_uc or a claim at legacy + closure; later stages rewrite would_claim_uc, and + ``uc_legacy_closure_take_up`` first writes the closure claim's input. So + the engine reads ``frame`` without the columns first written after the + BRMA stage (``_columns_first_written_after``), with every column an E4 + stage before it wrote and a later stage declares rewriting put back from + the recompute (``drawn``, keyed by entity id). A column a later stage + declares rewriting that the artifact cannot restore stays as stored; the + scope names each with the stages that rewrite it, salary_sacrifice among + them, whose reversal takes back only the records it converted. + """ + + from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + ) + + first_later = _columns_first_written_after(_BRMA_STAGE) + # The salary-sacrifice reversal takes back only the converted records' pay + # and contributions, not what the stage wrote on the others, so the + # columns it writes count as rewritten here and are named if carried. + rewritten = _rewritten_after( + _BRMA_STAGE, + reversed_stages=tuple( + stage for stage in reversed_stages if stage != _SALARY_SACRIFICE_STAGE + ), + ) + last_writer = _last_writers(at_or_before=_BRMA_STAGE) + id_columns = {"person": "person_id", "benunit": "benunit_id"} + id_columns["household"] = "household_id" + tables: dict[str, pd.DataFrame] = {} + dropped: set[str] = set() + restored: list[str] = [] + for entity, id_column in id_columns.items(): + table = frame.table(entity).copy() + drop = [column for column in table.columns if column in first_later] + dropped.update(drop) + table = table.drop(columns=drop) + ids = table[id_column].to_numpy() + for column in drawn[entity].columns: + if ( + column in rewritten + and column in table.columns + and last_writer.get(column) in _E4_STAGES + ): + table[column] = drawn[entity][column].reindex(ids).to_numpy() + restored.append(column) + tables[entity] = table + carried = { + column: list(stages) + for column, stages in sorted(rewritten.items()) + if column not in restored + and any(column in table.columns for table in tables.values()) + } + staged = uk_national_frame( + person=tables["person"], + benunit=tables["benunit"], + household=tables["household"], + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + lha_category = engine.materialize( + _engine_safe_frame(staged), ("LHA_category",), uk_time_period(frame) + )["LHA_category"] + scope = { + "stage": _BRMA_STAGE, + "columns_first_written_later": len(dropped), + "restored_from_the_recompute": sorted(restored), + "carried_as_stored": carried, + } + return lha_category, scope + + +def _apprentice_walk_holds( + person: pd.DataFrame, + flags: np.ndarray, + *, + person_weights: np.ndarray, + rate: float, +) -> bool: + """Whether stored ``is_apprentice`` flags are the stage's walk, to rounding. + + ``frs_person_draws`` flags the eligible employees aged 21 to 24 in the + order of their identity-keyed uniform until their weight lies nearest + ``rate`` times the weight of everyone aged 21 to 24 + (``derive_is_apprentice``). ``person_weights`` are each person's household + weight restored to the stage's to rounding (``_frame_as_draw_stages_saw``), + so where two prefixes of the walk lie within rounding of the target a + recompute can flag one person more or fewer than the stage did. The + stored flags hold where they are exactly the eligible employees whose + uniform lies at or below the largest flagged one's, and their weight lies + as near the target as the recompute's, to + :data:`_APPRENTICE_WALK_RELATIVE_TOLERANCE` of the weight of everyone + aged 21 to 24. + """ + + from microcosm.build.uk_runtime.frs_person_draws import ( + APPRENTICE_EMPLOYEE_STATUSES, + APPRENTICE_MAX_AGE_EXCLUSIVE, + APPRENTICE_MIN_AGE, + APPRENTICE_OUTPUT, + _draws, + _nearest_target_threshold, + ) + + flags = np.asarray(flags, dtype=bool) + weights = np.asarray(person_weights, dtype=float) + age = pd.to_numeric(person["age"], errors="coerce").to_numpy(dtype=float) + in_ages = (age >= APPRENTICE_MIN_AGE) & (age < APPRENTICE_MAX_AGE_EXCLUSIVE) + eligible = in_ages & ( + person["employment_status"] + .astype(str) + .isin(APPRENTICE_EMPLOYEE_STATUSES) + .to_numpy() + ) + uniforms = _draws(person["person_id"].to_numpy(), APPRENTICE_OUTPUT) + if (flags & ~eligible).any(): + return False + if flags.any() and not np.array_equal( + flags, eligible & (uniforms <= uniforms[flags].max()) + ): + return False + population = float(weights[in_ages].sum()) + target = float(rate) * population + threshold = _nearest_target_threshold( + target, draws=uniforms[eligible], weights=weights[eligible] + ) + walked = float(weights[eligible & (uniforms < threshold)].sum()) + stored = float(weights[flags].sum()) + return abs(stored - target) <= ( + abs(walked - target) + _APPRENTICE_WALK_RELATIVE_TOLERANCE * population + ) def e5_identity_receipt( @@ -235,8 +504,19 @@ def e5_identity_receipt( *, regional_resource: Mapping[str, object] | None = None, permutation_seed: int, + rewritten_later: Mapping[str, Sequence[str]] | None = None, + reversed_stages: Sequence[str] | None = None, ) -> dict[str, object]: - """Receipt E5 deterministic layers under row permutation by entity id.""" + """Receipt E5 deterministic layers under row permutation by entity id. + + A column a stage after its own declares writing, in the packaged manifest + (``rewritten_later``, by default ``_later_writers`` over the E5 stages, + the stages in ``reversed_stages`` excepted: by default those whose + rewrite ``_frame_as_stage_saw`` takes back), may hold the later stage's + value, so it is not compared with the store and the receipt names the + stages that wrote it: ``student_loans`` re-splits each household's + student loan balance after ``was_wealth`` allocated it. + """ resource = regional_resource or load_regional_land_values_resource() @@ -325,8 +605,17 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: ("household", "property_wealth"): (1e-12, 1e-6), ("person", "student_loan_balance"): (1e-12, 1e-6), } + if rewritten_later is None: + rewritten_later = _later_writers( + [column for values in original.values() for column in values.columns], + checked_stages=_E5_STAGES, + reversed_stages=( + _SCOPE_REVERSALS if reversed_stages is None else reversed_stages + ), + ) mismatches: dict[str, list[str]] = {} stored_mismatches: dict[str, list[str]] = {} + rewritten: dict[str, dict[str, list[str]]] = {} stored_tables = {"household": household, "person": person} for entity, values in original.items(): for column in values.columns: @@ -341,7 +630,9 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: ): mismatches.setdefault(entity, []).append(column) stored_table = stored_tables[entity] - if column in stored_table.columns: + if column in stored_table.columns and column in rewritten_later: + rewritten.setdefault(entity, {})[column] = list(rewritten_later[column]) + elif column in stored_table.columns: stored_rtol, stored_atol = stored_tolerances.get( (entity, column), (rtol, atol) ) @@ -359,6 +650,7 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: "permutation_mismatches": mismatches, "matches_stored_columns": not stored_mismatches, "stored_column_mismatches": stored_mismatches, + "rewritten_later": rewritten, "tolerance_policy": ( "permutation: bitwise for the regional-uprating rewrite " "(order-independent factor), rtol 1e-12 / atol 1e-6 GBP for the " @@ -366,7 +658,9 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: "/ atol 1e-6 GBP for both layers (re-applying uprating to the " "uprated fixed point and re-summing allocations each cost one " "float rounding generation, and the scoping's reversal of the " - "regional home values' wealth move one more); corporate_wealth " + "regional home values' wealth move one more), except a column a " + "stage after its own declares writing, which is not compared " + "(rewritten_later names the stages); corporate_wealth " "fold not reconstructible from the artifact (components consumed) - " "unit-tested only" ), @@ -1587,6 +1881,14 @@ def recompute(person_table: pd.DataFrame): return receipt +#: The relative tolerance E8 holds each SPI income band donor funding factor +#: it rebuilds from the stored weights to: the band-donor weights' own. The +#: rebuilt factors carry the stage's exact-total correction, which moves one +#: incumbent row by the last bits of the whole frame's total, so they are the +#: stage's to rounding, not to the stage gate's few ulps (microcosm#931). +_BAND_DONOR_FUNDING_RTOL = 1e-9 + + def _e8_band_donors( frame, problems: dict[str, object], @@ -1626,12 +1928,27 @@ def _e8_band_donors( recomputed. A row minted after the stage that the scoping leaves in place is refused by count, since the rerun would seat it as a household the stage saw. Problems land under ``band_donor_*`` keys. + + The receipt gives the layer in the forms of the stage's capped receipt, + at every seating (microcosm#931): the donors and each band's donors in + rounded distinct source households; each funding stratum with its donors + so counted and its masses and factor withheld, as the stage's funding + rows give them; and, for the funding, the stage's rule recomputed from + the stored weights (``funding_within_share``), every stratum within the + declared reallocation share. The checks compare the exact weights, + factors and seats; the receipt writes none of them. """ from microcosm.build.uk_runtime.national_frame import ( uk_household_weight_kind, uk_national_frame, ) + from microcosm.build.uk_runtime.receipt_counts import ( + UK_WITHHELD, + distinct_count, + rounded_count, + source_household_ids, + ) from microcosm.build.uk_runtime.spi_band_donors import ( HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, PERSON_IS_SPI_INCOME_BAND_CARRIER, @@ -1640,8 +1957,8 @@ def _e8_band_donors( SPI_INCOME_BAND_DONOR_LOWER_BOUNDS, SPI_INCOME_BAND_DONOR_MASS_CHANGE_REASON, SPI_INCOME_BAND_DONOR_SEED, + SPI_INCOME_BAND_MAXIMUM_REALLOCATION_SHARE, UK_SPI_INCOME_BAND_DONORS_STAGE_NAME, - _funding_strata, stack_spi_income_band_donors, ) from microcosm.build.uk_runtime.spi_support import SPI_SUPPORT_COPY_ID_FIELD @@ -1707,7 +2024,16 @@ def _e8_band_donors( .astype("int64") .to_numpy() ) - receipt["donor_households"] = int(is_donor.sum()) + # Each donor's true source household, which the receipt counts by, as the + # stage's receipt does: a clone's donor and a one-clone donor of the same + # survey household are one source (microcosm#931). + donor_sources = source_household_ids( + household.loc[is_donor], stage=UK_SPI_INCOME_BAND_DONORS_STAGE_NAME + ) + donor_bands = donor_band[is_donor] + receipt["donor_households"] = rounded_count( + distinct_count(donor_sources, np.ones(len(donor_sources), dtype=bool)) + ) stored_seats = pd.Series(donor_band[is_donor], index=donor_source).sort_index() # Stored-layer consistency. @@ -1731,7 +2057,11 @@ def _e8_band_donors( ) if without_one or len(carriers_per_household) != len(donor_ids): layer["donors_without_exactly_one_carrier"] = int(without_one) + # Each band's donors in rounded distinct source households, the form the + # stage's capped receipt gives them in; its one donor weight is held here + # and against the rerun below, never written. band_rows: list[dict[str, object]] = [] + band_weight: dict[int, float | None] = {} unequal = 0 for band in SPI_INCOME_BAND_DONOR_LOWER_BOUNDS: band_weights = weights[is_donor & (donor_band == band)] @@ -1739,12 +2069,13 @@ def _e8_band_donors( band_weights, band_weights[0], rtol=1e-9, atol=1e-6 ): unequal += 1 + band_weight[int(band)] = float(band_weights[0]) if band_weights.size else None band_rows.append( { "lower_bound": int(band), - "donor_households": int(band_weights.size), - "donor_weight": (float(band_weights[0]) if band_weights.size else None), - "weighted_taxpayers": float(band_weights.sum()), + "donor_households": rounded_count( + distinct_count(donor_sources, donor_bands == band) + ), } ) if unequal: @@ -1757,16 +2088,46 @@ def _e8_band_donors( # region, within its geographic clone where the frame carries clones) # gave up the stratum's donor mass in proportion, so scaling them back to # the stratum's whole mass restores them. - strata = _funding_strata(household).to_numpy() - region_mass = pd.Series(weights).groupby(strata).sum() - incumbent_mass = pd.Series(weights[~is_donor]).groupby(strata[~is_donor]).sum() - if (incumbent_mass.reindex(region_mass.index).fillna(0.0) <= 0.0).any(): + strata, factor = _band_donor_funding_factors(household, weights) + if factor is None: problems["band_donor_funding"] = "a donor stratum has no incumbent mass" return receipt - factor = incumbent_mass / region_mass.reindex(incumbent_mass.index) - receipt["funding"] = { - str(stratum): float(value) for stratum, value in factor.sort_index().items() - } + # The funding rows in the stage's form at every seating: each stratum and + # its donors in rounded distinct source households, its masses and factor + # withheld (microcosm#931), and the stage's funding rule for the factors. + donor_strata = strata[is_donor] + receipt["funding"] = [ + { + "stratum": str(stratum), + "donor_households": rounded_count( + distinct_count(donor_sources, donor_strata == stratum) + ), + "incumbent_mass": UK_WITHHELD, + "donor_mass": UK_WITHHELD, + "factor": UK_WITHHELD, + } + for stratum in sorted(factor.index, key=str) + ] + # Every stratum's incumbents keep from one less the declared share to all + # of its mass. Rebuilt from the stored weights, the factors carry the + # stage's exact-total correction, which lands on one incumbent row and + # can move a small stratum's factor by far more than the few ulps the + # stage holds its own factors to, so they are held at E8's weight + # tolerance (_BAND_DONOR_FUNDING_RTOL). + rebuilt = factor.to_numpy(dtype=float) + floor = 1.0 - SPI_INCOME_BAND_MAXIMUM_REALLOCATION_SHARE + within_share = bool( + ( + (rebuilt >= floor * (1.0 - _BAND_DONOR_FUNDING_RTOL)) + & (rebuilt <= 1.0 + _BAND_DONOR_FUNDING_RTOL) + ).all() + ) + receipt["funding_within_share"] = within_share + if not within_share: + problems["band_donor_funding_share"] = ( + "a funding stratum's incumbents kept less than one less the declared " + "reallocation share of its mass" + ) pre_weights = weights[~is_donor] / pd.Series(strata[~is_donor]).map( factor ).to_numpy(dtype=float) @@ -1843,8 +2204,9 @@ def recompute(person_t, household_t, weights_t): seats, carrier_ids, evidence = recompute(pre_person, pre_household, pre_weights) receipt["seating_recomputed"] = True + # Full or capped, as the stage's receipt says (microcosm#931). The mass + # scale enters the band weights compared below and is not written. receipt["seating_scale"] = evidence["seating_scale"] - receipt["mass_scale"] = evidence["mass_scale"] missing = int((~stored_seats.index.isin(seats.index)).sum()) extra = int((~seats.index.isin(stored_seats.index)).sum()) if missing or extra: @@ -1859,15 +2221,10 @@ def recompute(person_t, household_t, weights_t): int(row["lower_bound"]): float(row["donor_weight"]) for row in evidence["bands"] } weight_off = [ - int(row["lower_bound"]) - for row in band_rows - if row["donor_weight"] is None - or not np.isclose( - row["donor_weight"], - recomputed_weight[int(row["lower_bound"])], - rtol=1e-9, - atol=1e-6, - ) + band + for band, stored_weight in band_weight.items() + if stored_weight is None + or not np.isclose(stored_weight, recomputed_weight[band], rtol=1e-9, atol=1e-6) ] if weight_off: problems["band_donor_weights_stored"] = weight_off @@ -1885,6 +2242,34 @@ def recompute(person_t, household_t, weights_t): return receipt +def _band_donor_funding_factors( + household: pd.DataFrame, weights: np.ndarray +) -> tuple[np.ndarray, pd.Series | None]: + """Each household's funding stratum, and each stratum's funding factor. + + ``household`` and ``weights`` are the frame the SPI income band donor + stage wrote (``_frame_as_stage_saw``). The stage scaled every incumbent + of a stratum (a region, within its geographic clone where the frame + carries clones) by one factor and gave the rest of the stratum's mass to + its donors, so the incumbents' mass over the stratum's whole mass is that + factor, to rounding. The factor is None where a stratum holds no + incumbent mass, which no stage-written frame does. + """ + + from microcosm.build.uk_runtime.spi_band_donors import ( + HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR, + _funding_strata, + ) + + is_donor = household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR].astype(bool).to_numpy() + strata = _funding_strata(household).to_numpy() + stratum_mass = pd.Series(weights).groupby(strata).sum() + incumbent_mass = pd.Series(weights[~is_donor]).groupby(strata[~is_donor]).sum() + if (incumbent_mass.reindex(stratum_mass.index).fillna(0.0) <= 0.0).any(): + return strata, None + return strata, incumbent_mass / stratum_mass.reindex(incumbent_mass.index) + + def _band_donor_seating_inputs( frame, spi_tab: Path | None ) -> tuple[pd.DataFrame | None, dict[int, float] | None]: @@ -1940,9 +2325,11 @@ def e8_identity_receipt( recomputed in full (identity-keyed top-ups at the release calibration year) in original and permuted row order against the stored column; (4) the SPI income band donors (microcosm#1063) - the stored layer's - structure, equal band weights and conserving mass record, and, with the - licensed SPI tape (``spi_tab``), the identity-keyed seating rerun on the - reconstructed pre-donor frame in original and permuted row order; + structure, equal band weights, funding within the declared share and + conserving mass record, and, with the licensed SPI tape (``spi_tab``), + the identity-keyed seating rerun on the reconstructed pre-donor frame in + original and permuted row order, given in the forms of the stage's + capped receipt; (5) the CGT residential split (microcosm#1063) recomputed from the folded pre-split frame with the stage's own function - the solved probability carried as arm weights - in original and permuted row order against the @@ -2139,7 +2526,10 @@ def plans_equal(left: pd.Series, right: pd.Series) -> bool: "student_loan_plan: exact equality; " "band-donor seats, bands and " "carriers: exact equality; band-donor weights within a band and " - "against the rerun: rtol 1e-9 / atol 1e-6; band-donor mass " + "against the rerun: rtol 1e-9 / atol 1e-6; band-donor funding " + "factors: rebuilt from the stored weights, each from one less the " + "declared reallocation share to one at rtol 1e-9 (they carry the " + "stage's exact-total correction); band-donor mass " "record: new_total against old_total at rtol 1e-9 with declared " "factor one; residential-split arms, indices and residential " "gains: exact equality; residential-split weights against the " @@ -2308,6 +2698,120 @@ def recompute(benunit_t: pd.DataFrame) -> pd.DataFrame: } +#: The sidecar key the spine build records its geographic clone count under, +#: and the stage whose receipt records the count it counted its copies on +#: (microcosm#931). +_SIDECAR_CLONE_COUNT_KEY = "geographic_clone_count" +_CGT_SUPPORT_SPLIT_STAGE = "cgt_support_split" + + +def _read_sidecar(path: Path) -> dict[str, object]: + """The spine's build sidecar, refused unless it is a JSON object.""" + + try: + sidecar = json.loads(Path(path).read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise ValueError("The spine sidecar is not valid JSON.") from error + if not isinstance(sidecar, dict): + raise ValueError("The spine sidecar is not a JSON object.") + return sidecar + + +def _is_clone_count(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 1 + + +def geographic_clone_counts( + frame, + *, + sidecar: Mapping[str, object] | None = None, + input_h5: Path | None = None, +) -> dict[str, int | None]: + """The frame's geographic clone count ``K``, checked against its spine. + + The frame carries ``K`` as its household clone indices, 0 to ``K - 1`` + (``geographic_clones.geographic_clone_count``; a frame without them is + one clone). Given the spine's build ``sidecar``, ``K`` must also be the + count the spine build recorded (``geographic_clone_count``) and, where + the spine ran ``cgt_support_split``, the count that stage's receipt says + it counted its copies on; and a sidecar that records its spine's file + digest must name ``input_h5``. The receipts scale by ``K`` and E4's + engine refuses any other count, so the tool refuses a disagreement by + count, and a missing record or another file in a fixed sentence + (microcosm#931). Returns the counts by source, ``None`` for a stage the + spine did not run. + """ + + counts: dict[str, int | None] = { + "frame": geographic_clone_count(frame.table("household")) + } + if sidecar is None: + return counts + recorded_output = sidecar.get("output") + digest = ( + recorded_output.get("sha256") if isinstance(recorded_output, Mapping) else None + ) + if isinstance(digest, str) and input_h5 is not None: + hasher = hashlib.sha256() + with Path(input_h5).open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + hasher.update(block) + if hasher.hexdigest() != digest: + raise ValueError( + "The spine sidecar records another H5 file than --input-h5: its " + "output digest differs." + ) + recorded = sidecar.get(_SIDECAR_CLONE_COUNT_KEY) + if not _is_clone_count(recorded): + raise ValueError( + f"The spine sidecar records no {_SIDECAR_CLONE_COUNT_KEY}, so the " + "frame's geographic clone count cannot be checked against the " + "spine's (microcosm#931)." + ) + counts["sidecar"] = int(recorded) + stages = sidecar.get("stages") + if not isinstance(stages, list): + raise ValueError("The spine sidecar records no stage list.") + counts[_CGT_SUPPORT_SPLIT_STAGE] = None + if _CGT_SUPPORT_SPLIT_STAGE in stages: + evidence = sidecar.get("stage_evidence") + receipt = ( + evidence.get(_CGT_SUPPORT_SPLIT_STAGE) + if isinstance(evidence, Mapping) + else None + ) + parameters = receipt.get("parameters") if isinstance(receipt, Mapping) else None + stage_count = ( + parameters.get("clone_count") if isinstance(parameters, Mapping) else None + ) + if not _is_clone_count(stage_count): + raise ValueError( + f"The spine sidecar's {_CGT_SUPPORT_SPLIT_STAGE} receipt records " + "no clone_count, so the count it counted its copies on cannot be " + "checked (microcosm#931)." + ) + counts[_CGT_SUPPORT_SPLIT_STAGE] = int(stage_count) + if len({count for count in counts.values() if count is not None}) != 1: + stage = counts[_CGT_SUPPORT_SPLIT_STAGE] + if stage is None: + named = "The frame's geographic clone count and the spine sidecar's" + found = f"frame {counts['frame']}, sidecar {counts['sidecar']}" + else: + named = ( + "The frame's geographic clone count, the spine sidecar's and the " + f"one {_CGT_SUPPORT_SPLIT_STAGE} counted its copies on" + ) + found = ( + f"frame {counts['frame']}, sidecar {counts['sidecar']}, " + f"{_CGT_SUPPORT_SPLIT_STAGE} {stage}" + ) + raise ValueError( + f"{named} are not one count ({found}); every receipt reads one clone " + "count (microcosm#931)." + ) + return counts + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-h5", type=Path, required=True) @@ -2326,6 +2830,18 @@ def main() -> int: "it that layer is checked for structure and mass only." ), ) + parser.add_argument( + "--spine-sidecar", + type=Path, + default=None, + help=( + "The spine's build sidecar (.build.json). The tool then " + "checks the frame's geographic clone count against the sidecar's " + "and the one cgt_support_split counted its copies on, refusing by " + "count where they differ, and refuses a sidecar whose output " + "digest is not --input-h5's." + ), + ) args = parser.parse_args() frame, _provenance = load_uk_national_frame(args.input_h5) @@ -2336,26 +2852,39 @@ def main() -> int: # (microcosm#931); a spine minted by the rule the table replaced is # refused once, here, before any receipt reads it. _require_copy_ids_on_the_declared_table(frame.table("household")) + # The geographic clone count K every receipt reads, checked against the + # spine's sidecar where one is given (microcosm#931). + clones = geographic_clone_counts( + frame, + sidecar=None + if args.spine_sidecar is None + else _read_sidecar(args.spine_sidecar), + input_h5=args.input_h5, + ) if args.check == "e4": from microcosm.build.uk_runtime.take_up_contract import load_uk_take_up_contract from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine - # E4 draws ran on the pre-stack FRS spine and its take-up targets are + # E4's stages drew on every geographic clone of the raw FRS rows, + # before the SPI rows were stacked, and its take-up targets are # unweighted int(rate * n_units): recompute on the survey channel # only, or every target moves with the synthetic rows (the post-#717 - # scoping rule the E5 receipt already applies). - frame = _frs_only_frame(frame) - engine = PolicyEngineUKEngine() - lha_category = engine.materialize( - _engine_safe_frame(frame), ("LHA_category",), uk_time_period(frame) - )["LHA_category"] + # scoping rule the E5 receipt already applies), at the weights the + # stages read, as far as the artifact restores them. The engine runs + # clone by clone at the spine's clone count, as the stages' did. + population_policy = uk_take_up_population_policy(uk_time_period(frame)) receipt = e4_identity_receipt( - frame, + _frame_as_draw_stages_saw( + frame, state_pension_age=population_policy.state_pension_age + ), contract=load_uk_take_up_contract(), count_resource=load_brma_count_resource(), - lha_category=lha_category, + engine=PolicyEngineUKEngine(clones=clones["frame"]), permutation_seed=args.permutation_seed, - population_policy=uk_take_up_population_policy(uk_time_period(frame)), + population_policy=population_policy, + reversed_stages=_reversals_applied( + frame, UK_SPI_INCOME_BAND_DONORS_STAGE_NAME + ), ) ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] @@ -2369,10 +2898,11 @@ def main() -> int: # population the stage saw. regional_home_values then rewrites both # property columns, so the fixed point holds only on the values the # uprating left, which the scoping restores from that stage's rank - # basis. + # basis where the artifact keeps it. receipt = e5_identity_receipt( _frame_as_stage_saw(frame, _E5_FIRST_STAGE), permutation_seed=args.permutation_seed, + reversed_stages=_reversals_applied(frame, _E5_FIRST_STAGE), ) ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] @@ -2417,6 +2947,7 @@ def main() -> int: receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) receipt["input_h5"] = str(args.input_h5) + receipt["geographic_clones"] = clones args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") print( "identity stability:", @@ -2454,6 +2985,29 @@ def main() -> int: _E5_FIRST_STAGE = "was_wealth" _E6_FIRST_STAGE = "nts_bus_travel" +#: The stages each receipt recomputes, in the roster's order. A recomputed +#: column belongs to the last of them that declares writing it, and a stage +#: after that one that declares writing it again can leave the final frame +#: holding its value instead of the recomputed stage's (``_later_writers``). E4's +#: stages run after geographic_expand, on every geographic clone, and before +#: the SPI support channel stacks its copies (microcosm#931); E5 checks the +#: wealth stage's layers and the regional uprating's fixed point. +_E4_STAGES = ("frs_take_up", "frs_person_draws", "frs_household_draws", "frs_brma") +_E5_STAGES = ("was_wealth", "regional_property_uprating") +#: The stage whose LHA_category E4's BRMA draw reads +#: (``_brma_stage_lha_category``). +_BRMA_STAGE = "frs_brma" +#: The SPI support channel, whose prior-mass allocation E4's scope takes back +#: (``_frame_as_draw_stages_saw``). +_SPI_SUPPORT_STAGE = "spi_support_channel" +_SPI_PRIOR_MASS_OPERATION = "allocate_zero_weight_prior_mass" +#: How far, relative to the weight of everyone aged 21 to 24, stored +#: is_apprentice flags may lie further from the walk's target than the +#: recompute's on the restored weights (``_apprentice_walk_holds``): the +#: restored weights are the stage's to a few floats per row, and to the +#: exact-total corrections of two stages, far inside it. +_APPRENTICE_WALK_RELATIVE_TOLERANCE = 1e-9 + #: The stage that stacks each flag, for artifacts that carry it, when that #: stage's mass move is a uniform divisor (``_stage_time_weight_divisor``). #: The weight restoration is driven by what the *artifact* actually contains @@ -3153,6 +3707,251 @@ def _frame_as_stage_saw(frame, stage: str): ) +def _reversals_applied(frame, stage: str) -> tuple[str, ...]: + """The stages whose rewrite ``_frame_as_stage_saw(frame, stage)`` takes back. + + Each reversal reads a carrier the artifact keeps (the salary-sacrifice + stage's pre-conversion pay, the regional home values' rank basis), so an + artifact without it, a release say, keeps the later stage's values, and + a receipt reports that stage's columns as rewritten later. + """ + + from microcosm.build.uk_runtime.regional_home_values import RANK_BASIS_COLUMN + from microcosm.build.uk_runtime.salary_sacrifice import ( + SALSAC_PRE_CONVERSION_PAY_COLUMN, + ) + + positions = _roster_positions() + applied: list[str] = [] + if positions[stage] < positions[_SALARY_SACRIFICE_STAGE] and ( + SALSAC_PRE_CONVERSION_PAY_COLUMN in frame.table("person").columns + ): + applied.append(_SALARY_SACRIFICE_STAGE) + if positions[stage] < positions[_REGIONAL_HOME_VALUES_STAGE] and ( + RANK_BASIS_COLUMN in frame.table("household").columns + ): + applied.append(_REGIONAL_HOME_VALUES_STAGE) + return tuple(applied) + + +def _frame_as_draw_stages_saw(frame, *, state_pension_age: int | None): + """The FRS rows E4's stages drew on, at the weights they read. + + E4's stages ran on every geographic clone of the raw FRS rows, before the + SPI support channel stacked its copies, at the weights geographic_expand + left (microcosm#931). The rows are ``_frs_only_frame``'s. Their weights + are taken back in the roster's reverse order, as far as the artifact + allows: every layer stacked after the SPI income band donors folds back + onto the row it came from (``_frame_as_stage_saw``, which also reverses + the salary-sacrifice conversion); the band donors' funding is taken + back, every incumbent of a funding stratum divided by the stratum's + factor (``_band_donor_funding_factors``); and the support channel's + prior mass is taken back, every base household divided by one less the + share the packaged manifest declares for it (``_spi_prior_mass_shares``), + the pension-age share where a member is at or over ``state_pension_age`` + and the share otherwise. Neither move is uniform: the two shares differ, + and so do the strata's factors. The folds re-add floats, each move + divides in floats, and the stages that move mass correct a row each to + an exact total, so the weights are the stages' to rounding, not bit for + bit (``_apprentice_walk_holds`` holds the one draw that reads them to + that). An artifact without either layer skips its step. + """ + + from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + ) + from microcosm.build.uk_runtime.spi_support import HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN + + staged = _frame_as_stage_saw(frame, UK_SPI_INCOME_BAND_DONORS_STAGE_NAME) + household = staged.table("household") + person = staged.table("person") + weights = np.asarray(staged.weights_for("household").values, dtype=float).copy() + keep = np.ones(len(household), dtype=bool) + if HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR in household.columns: + donor = household[HOUSEHOLD_IS_SPI_INCOME_BAND_DONOR].astype(bool).to_numpy() + strata, factor = _band_donor_funding_factors(household, weights) + if factor is None: + raise ValueError( + "A funding stratum of the SPI income band donors holds no " + "incumbent mass, so E4's scope cannot take their funding back." + ) + weights[~donor] = weights[~donor] / pd.Series(strata[~donor]).map( + factor + ).to_numpy(dtype=float) + keep &= ~donor + if HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN in household.columns: + synthetic = household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN].astype(bool).to_numpy() + share, pension_age_share = _spi_prior_mass_shares() + # The share of each base household's mass the channel moved to its + # SPI copies; the household kept one less it. + moved_share = np.full(len(household), share) + if pension_age_share is not None: + if state_pension_age is None: + raise ValueError( + "The SPI support channel's pension-age share needs the State " + "Pension age it split the strata at; E4's scope takes it from " + "the take-up population policy." + ) + # The ages of the rows the scope keeps, which the channel read and + # refused unless numeric, as this does in a fixed sentence. + base = set(household.loc[keep & ~synthetic, "household_id"]) + members = person.loc[person["person_household_id"].isin(base)] + ages = pd.to_numeric(members["age"], errors="coerce") + if ages.isna().any(): + raise ValueError( + "person.age must be numeric to take the SPI support channel's " + "pension-age share back." + ) + older = members.loc[ages.ge(int(state_pension_age)), "person_household_id"] + pension_age = household["household_id"].isin(set(older)).to_numpy() + moved_share = np.where(pension_age, pension_age_share, share) + weights[~synthetic] = weights[~synthetic] / (1.0 - moved_share[~synthetic]) + keep &= ~synthetic + kept = household.loc[keep].reset_index(drop=True) + kept_ids = set(kept["household_id"].tolist()) + kept_person = person.loc[person["person_household_id"].isin(kept_ids)] + benunit = staged.table("benunit") + return uk_national_frame( + person=kept_person.reset_index(drop=True), + benunit=benunit.loc[ + benunit["benunit_id"].isin(set(kept_person["person_benunit_id"])) + ].reset_index(drop=True), + household=kept, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(staged), + household_weights=weights[keep], + mass_log=staged.mass_log, + ) + + +def _spi_prior_mass_shares() -> tuple[float, float | None]: + """The SPI support channel's declared prior-mass share, and its pension-age + share where it declares one, from the packaged manifest.""" + + (stage,) = [ + stage + for stage in _source_stage_roster() + if stage.get("stage") == _SPI_SUPPORT_STAGE + ] + (operation,) = [ + operation + for operation in stage.get("operations", ()) + if operation.get("kind") == _SPI_PRIOR_MASS_OPERATION + ] + pension_age_share = operation.get("pension_age_share") + return float(operation["share"]), ( + None if pension_age_share is None else float(pension_age_share) + ) + + +def _roster_writes() -> tuple[tuple[str, ...], tuple[frozenset[str], ...]]: + """The packaged roster's stage names and, per stage, the columns it + declares writing: its outputs and its rewrites.""" + + stages = _source_stage_roster() + return ( + tuple(str(stage["stage"]) for stage in stages), + tuple( + frozenset( + str(column) + for key in ("outputs", "rewrites") + for column in stage.get(key) or () + ) + for stage in stages + ), + ) + + +def _later_writers( + columns: Sequence[str], + *, + checked_stages: Sequence[str], + reversed_stages: Sequence[str] = (), +) -> dict[str, tuple[str, ...]]: + """The stages after each column's own that declare writing it again. + + A recomputed column's own stage is the last of ``checked_stages`` (the + stages a receipt recomputes) whose entry in the packaged manifest declares + it among its outputs or rewrites. Every stage after that one that declares + it too can leave the final frame holding its value, so the receipt does + not compare the column with the store (microcosm#931); the stages in + ``reversed_stages``, whose rewrite the receipt's scope took back, leave + the own stage's value and are not counted. Columns no later stage writes + are left out. A column no checked stage declares is refused: the receipt + could not place it. + """ + + names, writes = _roster_writes() + own_positions = sorted(names.index(stage) for stage in checked_stages) + skipped = set(reversed_stages) + later: dict[str, tuple[str, ...]] = {} + for column in columns: + declaring = [ + position for position in own_positions if column in writes[position] + ] + if not declaring: + raise ValueError( + f"No stage of {list(checked_stages)} declares writing {column!r} in " + "the packaged manifest, so the receipt cannot place it." + ) + writers = tuple( + names[position] + for position in range(declaring[-1] + 1, len(names)) + if column in writes[position] and names[position] not in skipped + ) + if writers: + later[column] = writers + return later + + +def _last_writers(*, at_or_before: str) -> dict[str, str]: + """Each column a stage up to ``at_or_before`` declares writing, mapped to + the last such stage: the stage whose value ``at_or_before`` read.""" + + names, writes = _roster_writes() + last: dict[str, str] = {} + for position in range(names.index(at_or_before) + 1): + for column in writes[position]: + last[column] = names[position] + return last + + +def _rewritten_after( + stage: str, *, reversed_stages: Sequence[str] = () +) -> dict[str, tuple[str, ...]]: + """The columns ``stage`` read that a later stage declares writing again. + + A column a stage up to ``stage`` declares writing, mapped to the later + stages that declare it too, the stages in ``reversed_stages`` excepted. + """ + + names, writes = _roster_writes() + end = names.index(stage) + 1 + seen = set().union(*writes[:end]) + skipped = set(reversed_stages) + rewritten: dict[str, tuple[str, ...]] = {} + for column in sorted(seen): + writers = tuple( + names[position] + for position in range(end, len(names)) + if column in writes[position] and names[position] not in skipped + ) + if writers: + rewritten[column] = writers + return rewritten + + +def _columns_first_written_after(stage: str) -> frozenset[str]: + """The columns a later stage declares writing that no stage up to + ``stage`` declares: columns the frame ``stage`` read did not carry.""" + + names, writes = _roster_writes() + end = names.index(stage) + 1 + seen = set().union(*writes[:end]) + return frozenset(set().union(*writes[end:]) - seen) + + #: The stage whose conversion rewrites pay in place (microcosm#1069 c9). _SALARY_SACRIFICE_STAGE = "salary_sacrifice" @@ -3249,6 +4048,10 @@ def _pre_salary_sacrifice_frame(frame): _REGIONAL_HOME_VALUES_STAGE = "regional_home_values" #: The household columns that stage rewrites, which the reversal restores. _REGIONAL_HOME_VALUE_COLUMNS = ("main_residence_value", "property_wealth") +#: The stages whose rewrite ``_frame_as_stage_saw`` takes back for a stage +#: before them, when the artifact keeps the carrier (``_reversals_applied``): +#: what a receipt assumes of a frame it is handed scoped. +_SCOPE_REVERSALS = (_SALARY_SACRIFICE_STAGE, _REGIONAL_HOME_VALUES_STAGE) def _source_stage_roster() -> list[dict[str, object]]: From 9ace5e265e4bdfaabfd65018c19bbf34e7c22915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:04:58 +0000 Subject: [PATCH 062/108] Keep stage 16's refusal of a clone stratum without an SPI copy (#931) Group (b)'s last commit: the small-clone-stratum rule, set from the f010, K = 15 arm as ruled on 7 Nov. Stage numbers keep the order before stages 14 and 15 moved. That arm (seed 578, at d86ec9322) drew an SPI copy in every clone stratum at stage 16 (spi_support_channel), so the rule stays "refuse as now": a clone's stratum with base mass and no SPI copy refuses the stage, as a one-clone spine's stratum would, and is never given its base mass back. A fuller rung draws more copies in each stratum, since the share copies each household with about the same probability at every rung. Stage 17's (spi_income_band_donors) common mass scale at K = 15 equals the f010, K = 1 arm's, so stage 17 is unchanged. - spi_support: the allocation's and the entry point's docstrings state the rule at every clone count. - A stage-level test: the stage refuses, by count, when its share leaves one clone's strata without a copy, and runs when it does not. Code-only, and no value moves. The docstrings move the implementation hashes of the ten UK stage kernels that import spi_support (stages 16 and 17 among them) and seven full-build kernels; no transport kernel and no phase code moves. The stage-16 notes sentence on the refusal rides (c)'s first regeneration, so (b) regenerates H2 once. Co-Authored-By: Claude Opus 5.5 --- .../microcosm/build/uk_runtime/spi_support.py | 10 ++++-- .../tests/engine_free/uk/test_uk_spi_spine.py | 34 ++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py index df98ffa313..20c61d02fc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_support.py @@ -606,7 +606,8 @@ def build_uk_spi_support_channel( and the channel's households are drawn unweighted with one weight per region, so without the split the channel holds more pension-age mass than the FRS design gives pension-age households (microcosm#1069). Where the - frame carries geographic clones, the strata split by clone as well + frame carries geographic clones, the strata split by clone as well, and a + clone's stratum that the share left without a copy refuses the stage (:func:`_allocate_spi_prior_mass`). """ @@ -1134,7 +1135,12 @@ def _allocate_spi_prior_mass( is derived for the allocation only and never written to the frame. A household table without the column allocates exactly as before, and so does a one-clone spine, whose clone stratum is the same for every - household. A positive-mass stratum with no SPI copy is refused, by count. + household. A positive-mass stratum with no SPI copy is refused, by count, + at every clone count: a clone's stratum that the share left without a + copy is not given its base mass back, which would leave that clone short + of its SPI share (microcosm#931). The f010, K = 15 arm drew a copy in + every clone stratum, and a fuller rung draws more in each, since the share + copies each household with about the same probability at every rung. """ household = result.household.copy() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py index 9bed95ac24..6b2bbb4c2d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spi_spine.py @@ -8,7 +8,7 @@ import pytest from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.uk_runtime import spi_income, spi_spine +from microcosm.build.uk_runtime import spi_income, spi_spine, spi_support from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.geographic_expand import expand_uk_geographic_clones from microcosm.build.uk_runtime.hmrc_income import ( @@ -1355,6 +1355,38 @@ def test_stage_16_counts_its_pension_age_households_in_sources_at_any_clone_coun ) +def test_stage_16_refuses_a_clone_stratum_its_share_left_without_a_copy( + monkeypatch, +) -> None: + """At any clone count, a clone's stratum with base mass and no SPI copy + refuses the stage, as a one-clone spine's stratum would; it is never + given its base mass back (microcosm#931). Here the share never draws + clone 2's London households, while every other clone and region keeps + its draws.""" + + frame = expand_uk_geographic_clones(_share_frame(), clones=3).frame + household = frame.table("household") + emptied = household.loc[ + household["household_clone_index"].eq(2) & household["region"].eq("LONDON"), + "household_id", + ].to_numpy() + drawn = spi_support.spi_share_inclusion + + def without_clone_two_london(household_ids, **arguments): + return drawn(household_ids, **arguments) & ~np.isin(household_ids, emptied) + + # Unpatched, the share draws a copy in every clone stratum. + _share_stage(frame) + monkeypatch.setattr(spi_support, "spi_share_inclusion", without_clone_two_london) + with pytest.raises(ValueError) as caught: + _share_stage(frame) + # London's two strata in clone 2, its pension-age households and the rest. + assert str(caught.value) == ( + "Every positive-mass UK base stratum must have rebuilt SPI support; 2 of " + "the 24 positive-mass strata have no SPI copy." + ) + + def test_the_support_channel_writes_no_source_column_of_its_own() -> None: """The households' source columns are geographic_expand's (microcosm#931). From 4c341b19bb03104f4c8f8554904ad01b6f36e65c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:34:08 +0000 Subject: [PATCH 063/108] Add a shared fallback-level chooser for conditioned area draws (#931) atomic_support_conditioning gains FallbackLevel and choose_fallback_levels(support, households, *, constraint_columns, weight_by, default_weight, levels). Each household takes the first level of a declared ladder whose inputs are present (and, where the level samples by the household's own weight, a code the rule maps) and whose support cell has positive summed mass in that weight. A household no level takes is refused with a fixed text that names no count, household or value. It is vectorised, with one grouped mass table per level, and names no country. Why: the geography-first draw conditions each household's area on its deprivation decile and council-tax band, and the shared draw refuses an empty cell, so the conditioned draw falls back along a ladder (the plan of 4 Nov and decision 9's fallback order: the band's weight before households, then the decile within the region before the region alone). The chooser is shared, for Max's review; nothing calls it yet. What moves: node keys only where a node's derived reach includes the module, and none does. A code-only probe found no transport kernel, UK graph kernel, gate battery phase or binding closure that hashes it, and no key moved when its bytes changed; no US module imports it. No output moves. Tests: test_atomic_support_conditioning.py, on invented supports: level 0 where the band has mass, level 1 where only households do, a missing decile falling to the region levels, the count-free refusal, the vectorised choice equal to a per-row loop, and Hypothesis properties that the chooser equals that loop and that the shared draw finds mass at the chosen level and at no earlier one. Co-Authored-By: Claude Opus 5.5 --- ...931-shared-fallback-level-chooser.added.md | 14 + .../build/atomic_support_conditioning.py | 230 +++++++++- .../test_atomic_support_conditioning.py | 429 +++++++++++++++++- 3 files changed, 671 insertions(+), 2 deletions(-) create mode 100644 changelog.d/931-shared-fallback-level-chooser.added.md diff --git a/changelog.d/931-shared-fallback-level-chooser.added.md b/changelog.d/931-shared-fallback-level-chooser.added.md new file mode 100644 index 0000000000..a9bdcf105c --- /dev/null +++ b/changelog.d/931-shared-fallback-level-chooser.added.md @@ -0,0 +1,14 @@ +A country-neutral fallback-level chooser for conditioned area draws: `FallbackLevel` and `choose_fallback_levels` in `microcosm.build.atomic_support_conditioning` (#931; shared, for Max's review). The shared draw (`assign_atomic`) refuses a household whose constraint cell has no sampling mass in the weight it samples by. A country that conditions its draw, as the UK geography-first draw will on the deprivation decile and the council-tax band, declares a fallback ladder instead, and the chooser picks each household's level before the draw. + +- **Declarations.** `FallbackLevel(name, constraints, weight_by)` names the household inputs a level keeps as exact-match constraints, and says whether the level samples by the weight chosen for each household (`weight_by=True`) or by the default weight. `choose_fallback_levels(support, households, *, constraint_columns, weight_by, default_weight, levels)` takes the decoded support, a frame of household inputs, the support code column each input must match, the per-household rule `{"input": ..., "map": {code: weight column}}` (the draw's `weight_by` without its default) or `None`, the default weight column, and the ladder in order. +- **The rule.** Each household takes the first level whose inputs are present (every constraint the level keeps and, where it samples by the household's own weight, a code the rule maps) and whose cell, the support's areas matching the household on every kept constraint, has positive summed mass in the weight the household samples by there. It returns each household's level name, in row order. A draw that keeps a household's level's inputs, reads every other input as missing and samples each stage by the same rule then meets no empty cell. The chooser is vectorised, with one grouped mass table per level, read for the households no earlier level took. +- **Refusals.** + - A household no level takes is refused with one fixed text that names no count, household or value. + - A present input that is not a nonempty string without surrounding whitespace is refused, as the draw refuses it. + - A declaration is refused where a level keeps an input `constraint_columns` does not map, a level samples by the household's own weight but no rule is given, an input maps to the area or to a column that is not a support code column, two inputs map to one column, a weight is not a support weight column, or two levels share a name. +- **No caller yet, and nothing moves.** The chooser names no country, and nothing calls it until the UK geography-first draw. No node binds the module: no transport kernel (the shared kernels New Zealand's graph runs on, the atomic geography and rules kernels among them), no UK graph kernel or gate battery phase, and no transport binding's source closure hashes it, and no US module imports it. So no node key moves. +- **Tests.** `tests/engine_free/shared/test_atomic_support_conditioning.py`, on invented supports only: + - a household whose band has mass in its cell takes the first level, one whose cell has households but no banded mass takes the second, and a missing decile falls to the region-only levels; + - a household no level takes is refused with the count-free text; + - on 3,000 invented households the vectorised choice equals a per-row loop over the support's arrays, in any row order; + - two Hypothesis properties on small invented supports and ladders: the chooser equals the per-row loop, and the shared draw itself finds mass at the chosen level and is refused at every earlier level whose inputs are present (and at every level, for a household the chooser refuses). diff --git a/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py b/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py index b84c9e1f18..9ce67faaba 100644 --- a/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py +++ b/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py @@ -15,7 +15,10 @@ - :func:`extend_atomic_support` adds declared columns to a support's bytes; - :func:`zero_mass_cells` lists every constraint cell, at each fallback level, where a weight column has no sampling mass, so a draw can choose its - fallback ahead of time rather than fail on an empty cell. + fallback ahead of time rather than fail on an empty cell; +- :func:`choose_fallback_levels` makes that choice for each household: the + first level of a declared ladder (:class:`FallbackLevel`) whose cell has + mass in the weight the household samples by there. """ from __future__ import annotations @@ -306,3 +309,228 @@ def zero_mass_cells( empty.append(list(cell)) table.append({"constraints": list(level), "weight": weight, "cells": empty}) return table + + +def _name(value: object) -> bool: + return isinstance(value, str) and bool(value) and value.strip() == value + + +@dataclass(frozen=True) +class FallbackLevel: + """One level of a conditioned draw's fallback ladder. + + ``constraints`` names the household inputs the level keeps as exact-match + constraints, each a key of :func:`choose_fallback_levels`' + ``constraint_columns``. ``weight_by`` says whether the level samples by + the weight column chosen for each household (``True``) or by the default + weight (``False``). A draw at the level reads every input it does not keep + as missing. + """ + + name: str + constraints: tuple[str, ...] + weight_by: bool + + def __post_init__(self) -> None: + _require(_name(self.name), "a fallback level needs a name") + _require( + type(self.constraints) is tuple + and all(_name(column) for column in self.constraints) + and len(set(self.constraints)) == len(self.constraints), + "a fallback level keeps a tuple of distinct input columns", + ) + _require(type(self.weight_by) is bool, "a fallback level's weight_by is a bool") + + +def _observed(series: pd.Series) -> tuple[np.ndarray, np.ndarray]: + """Where a household input is present, and the input as Python objects.""" + present = ~series.isna().to_numpy(dtype=bool) + codes = series.to_numpy(dtype=object) + _require( + all(_name(code) for code in codes[present]), + "present input codes must be nonempty strings", + ) + return present, codes + + +def _cell_mass( + support: AtomicSupport, columns: Sequence[str], weights: Sequence[str] +) -> tuple[pd.MultiIndex | None, np.ndarray]: + """The support's cells over ``columns``, each with its summed weights. + + Row ``i`` of the mass table is cell ``i``'s total in each of ``weights``, + in order. Without columns the whole support is one cell (``None``). + Totals stay exact: the decoder holds each weight column's total within + the exact sampling range. + """ + values = np.stack( + [np.asarray(support.arrays[weight], dtype=np.int64) for weight in weights], + axis=1, + ) + if not columns: + return None, values.sum(axis=0, keepdims=True) + codes, cells = pd.MultiIndex.from_arrays( + [support.arrays[column] for column in columns] + ).factorize() + mass = np.zeros((len(cells), len(weights)), dtype=np.int64) + np.add.at(mass, codes, values) + return cells, mass + + +def choose_fallback_levels( + support: AtomicSupport, + households: pd.DataFrame, + *, + constraint_columns: Mapping[str, str], + weight_by: Mapping[str, object] | None, + default_weight: str, + levels: Sequence[FallbackLevel], +) -> np.ndarray: + """Each household's first fallback level whose support cell has mass. + + The shared draw (:func:`~microcosm.build.atomic_geography.assign_atomic`) + refuses a household whose constraint cell has no sampling mass in the + weight column it samples by. A country that conditions its draw declares + a fallback ladder, ``levels``, in order. A household takes the first level + whose inputs are present (every constraint the level keeps and, where the + level samples by the household's own weight, a code ``weight_by`` maps) + and whose cell, the support's areas matching the household on every + constraint the level keeps, has positive summed weight in the column the + household samples by there. A household no level takes is refused; the + refusal names no count, household or value. + + ``constraint_columns`` maps each household input a level may keep to the + support code column it must match. ``weight_by`` is ``None`` or + ``{"input": column, "map": {code: weight column}}``: a level whose + ``weight_by`` is true samples by the column ``map`` names for the + household's ``input`` code. Every other level samples by + ``default_weight``, the draw's ``weight_by`` default or its one + ``weight``. A present input must be a nonempty string without surrounding + whitespace, as the draw requires. A draw that keeps a household's level's + inputs, reads every other input as missing and samples each stage by this + rule then meets no empty cell. + + Vectorised: one grouped mass table per level, read for the households no + earlier level took. Returns each household's level name, in row order. + """ + _require( + isinstance(support, AtomicSupport), "support must be a decoded atomic support" + ) + _require(isinstance(households, pd.DataFrame), "households must be a DataFrame") + columns = support.metadata["columns"] + + def is_weight(name: object) -> bool: + return _name(name) and columns.get(name, {}).get("kind") == "weight" + + _require( + isinstance(constraint_columns, Mapping) + and all( + _name(column) + and _name(matched) + and matched != "area" + and columns.get(matched, {}).get("kind") == "code" + for column, matched in constraint_columns.items() + ) + and len(set(constraint_columns.values())) == len(constraint_columns), + "constraint_columns must map inputs to distinct support code columns " + "other than the area", + ) + _require( + is_weight(default_weight), "default_weight must be a support weight column" + ) + _require( + weight_by is None + or ( + isinstance(weight_by, Mapping) + and set(weight_by) == {"input", "map"} + and _name(weight_by["input"]) + and isinstance(weight_by["map"], Mapping) + and bool(weight_by["map"]) + and all( + _name(code) and is_weight(weight) + for code, weight in weight_by["map"].items() + ) + ), + "weight_by must map codes of one input to support weight columns", + ) + _require( + isinstance(levels, Sequence) + and bool(levels) + and all(isinstance(level, FallbackLevel) for level in levels), + "levels must be a nonempty sequence of FallbackLevel", + ) + levels = tuple(levels) + _require( + len({level.name for level in levels}) == len(levels), + "fallback level names must be distinct", + ) + _require( + all(set(level.constraints) <= set(constraint_columns) for level in levels), + "a fallback level keeps an input constraint_columns does not map", + ) + weighted = any(level.weight_by for level in levels) + _require( + weight_by is not None or not weighted, + "a fallback level samples by weight_by, but no weight_by is given", + ) + inputs = {column for level in levels for column in level.constraints} + if weighted: + inputs.add(weight_by["input"]) + missing = sorted(inputs - set(households.columns)) + _require(not missing, f"households lack input columns {missing}") + observed = {column: _observed(households[column]) for column in sorted(inputs)} + + # Each household's own weight, as a position in the rule's weight columns + # (-1 where its code is missing or unmapped). + rows = len(households) + rule_weights: tuple[str, ...] = () + own_weight = np.full(rows, -1, dtype=np.int64) + if weighted: + rule = weight_by["map"] + rule_weights = tuple(dict.fromkeys(rule.values())) + present, codes = observed[weight_by["input"]] + mapped = pd.Index(list(rule)).get_indexer(codes[present]) + column_of_code = np.asarray( + [rule_weights.index(rule[code]) for code in rule], dtype=np.int64 + ) + holders = np.flatnonzero(present) + own_weight[holders[mapped >= 0]] = column_of_code[mapped[mapped >= 0]] + + chosen = np.full(rows, -1, dtype=np.int64) + for position, level in enumerate(levels): + open_rows = chosen < 0 + for column in level.constraints: + open_rows &= observed[column][0] + if level.weight_by: + open_rows &= own_weight >= 0 + candidates = np.flatnonzero(open_rows) + if not len(candidates): + continue + cells, mass = _cell_mass( + support, + [constraint_columns[column] for column in level.constraints], + rule_weights if level.weight_by else (default_weight,), + ) + if cells is None: + cell = np.zeros(len(candidates), dtype=np.intp) + else: + cell = cells.get_indexer( + pd.MultiIndex.from_arrays( + [observed[column][1][candidates] for column in level.constraints] + ) + ) + weight = ( + own_weight[candidates] + if level.weight_by + else np.zeros(len(candidates), dtype=np.int64) + ) + found = cell >= 0 + has_mass = np.zeros(len(candidates), dtype=bool) + has_mass[found] = mass[cell[found], weight[found]] > 0 + chosen[candidates[has_mass]] = position + _require( + bool(np.all(chosen >= 0)), + "a household has no fallback level whose inputs are present and whose " + "cell has sampling mass", + ) + return np.asarray([level.name for level in levels], dtype="U")[chosen] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_atomic_support_conditioning.py b/packages/microcosm-build/tests/engine_free/shared/test_atomic_support_conditioning.py index 15ac747096..44404427ad 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_atomic_support_conditioning.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_atomic_support_conditioning.py @@ -1,9 +1,13 @@ -"""The country-neutral columns a support carries for a conditioned draw.""" +"""The country-neutral columns a support carries for a conditioned draw, and +the fallback level each household draws at.""" from __future__ import annotations import numpy as np +import pandas as pd import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st from microcosm.build.atomic_geography import ( assign_atomic, @@ -12,7 +16,9 @@ validate_assignment_spec, ) from microcosm.build.atomic_support_conditioning import ( + FallbackLevel, apportion_weights, + choose_fallback_levels, extend_atomic_support, modal_area_codes, zero_mass_cells, @@ -305,3 +311,424 @@ def test_the_shared_operator_draws_on_the_added_columns(): # area with households. assert assigned["area_code"].tolist()[:2] == ["a2", "b1"] assert assigned["area_code"].tolist()[2] in {"a1", "a2"} + + +# The fallback-level chooser. Every support and household below is invented. + +#: A ladder in a conditioned draw's fallback order: the band's weight before +#: households, then the decile with the region before the region alone. +LADDER = ( + FallbackLevel("L0", ("region", "draw_decile"), weight_by=True), + FallbackLevel("L1", ("region", "draw_decile"), weight_by=False), + FallbackLevel("L2", ("region",), weight_by=True), + FallbackLevel("L3", ("region",), weight_by=False), +) +#: Each household input to the support column it must match. +CONSTRAINTS = {"region": "region", "draw_decile": "decile"} +RULE = {"input": "band", "map": {"x": "households_x", "y": "households_y"}} +NO_LEVEL = ( + "Atomic support conditioning: a household has no fallback level whose " + "inputs are present and whose cell has sampling mass" +) + + +def _conditioned(region, decile, **weights): + """Invented areas, each with a region, a decile and the given weights.""" + code = {"kind": "code", "source": "invented", "vintage": "v1", "relation": "exact"} + return decode_atomic_support( + encode_atomic_support( + { + "version": 1, + "system": "toy", + "level": "area", + "code_system": "toy", + "vintage": "v1", + "columns": { + "area": code, + "region": code, + "decile": code, + **{ + name: {"kind": "weight", "source": "invented", "basis": name} + for name in weights + }, + }, + }, + { + "area": _u(*(f"a{index}" for index in range(len(region)))), + "region": _u(*region), + "decile": _u(*decile), + **{ + name: np.asarray(values, dtype=np.int64) + for name, values in weights.items() + }, + }, + ) + ) + + +#: Five areas: band x has mass only in a0 (north, 1) and band y only in a2 +#: (south, 1); a1 (north, 2) has households but no banded mass; a4, the fen's +#: one area, has no mass in any column. +LADDER_SUPPORT = _conditioned( + region=("north", "north", "south", "south", "fen"), + decile=("1", "2", "1", "2", "1"), + households=(10, 30, 20, 0, 0), + households_x=(5, 0, 0, 0, 0), + households_y=(0, 0, 7, 0, 0), +) + + +def _households(*rows) -> pd.DataFrame: + """Households as (region, decile, band); None is a missing input.""" + return pd.DataFrame( + list(rows), columns=["region", "draw_decile", "band"], dtype=object + ) + + +def _choose(households, support=LADDER_SUPPORT, **overrides) -> list[str]: + arguments = { + "constraint_columns": CONSTRAINTS, + "weight_by": RULE, + "default_weight": "households", + "levels": LADDER, + **overrides, + } + return choose_fallback_levels(support, households, **arguments).tolist() + + +def _level_weight(household, level, weight_by, default_weight="households"): + """The weight column a household samples by at ``level``, or None where + an input the level reads is missing or, for its own weight, unmapped.""" + if any(pd.isna(household[column]) for column in level.constraints): + return None + if not level.weight_by: + return default_weight + code = household[weight_by["input"]] + return None if pd.isna(code) else weight_by["map"].get(code) + + +def _per_row(support, households, *, weight_by, levels, constraint_columns=CONSTRAINTS): + """The chooser's rule one household and one level at a time, on boolean + masks of the support's arrays: each household's first level whose inputs + are present and whose cell has mass, or None where some household has + none.""" + chosen = [] + for _, household in households.iterrows(): + for level in levels: + weight = _level_weight(household, level, weight_by) + if weight is None: + continue + cell = np.ones(len(support.arrays["area"]), dtype=bool) + for column in level.constraints: + cell &= support.arrays[constraint_columns[column]] == household[column] + if int(support.arrays[weight][cell].sum()) > 0: + chosen.append(level.name) + break + else: + return None + return chosen + + +def test_a_household_whose_band_has_mass_in_its_cell_takes_level_zero(): + rows = _households(("north", "1", "x"), ("south", "1", "y")) + assert _choose(rows) == ["L0", "L0"] + + +def test_a_household_whose_cell_has_only_households_takes_level_one(): + # a1 (north, 2) has no band-x mass; band z is a code the rule does not map, + # so its household skips both band-weighted levels. + rows = _households(("north", "2", "x"), ("north", "1", "z")) + assert _choose(rows) == ["L1", "L1"] + + +def test_a_missing_decile_falls_to_the_region_levels(): + rows = _households( + ("north", None, "x"), # band x has mass in the north: L2 + ("south", None, "x"), # but none in the south: L3 + ("north", None, None), # no band either: L3 + ("north", "1", None), # a decile and no band: L1 + ) + assert _choose(rows) == ["L2", "L3", "L3", "L1"] + + +@pytest.mark.parametrize( + "row", + [ + ("fen", "1", "x"), # the fen's area has no mass in any column + ("moor", "1", "x"), # no area in the moor + (None, "1", "x"), # every level keeps the region + ], +) +def test_a_household_no_level_takes_is_refused_naming_no_count_or_value(row): + with pytest.raises(ValueError) as refused: + _choose(_households(("north", "1", "x"), row, ("south", None, None))) + assert str(refused.value) == NO_LEVEL + + +def test_without_households_the_chooser_returns_no_level(): + assert _choose(_households()) == [] + + +def test_the_vectorised_choice_equals_a_per_row_loop(): + rng = np.random.default_rng(931) + areas, rows = 60, 3000 + counts, band_x, band_y = (rng.integers(0, 4, areas) for _ in range(3)) + # Many areas have no banded mass; every region keeps households, and every + # weight column has some mass. + band_x *= rng.integers(0, 2, areas) + band_y *= rng.integers(0, 2, areas) + counts[:3] += 1 + band_x[0] += 1 + band_y[1] += 1 + support = _conditioned( + region=np.tile(["r1", "r2", "r3"], areas // 3), + decile=rng.choice(["1", "2", "3", "4"], areas), + households=counts, + households_x=band_x, + households_y=band_y, + ) + frame = pd.DataFrame( + { + "region": rng.choice(["r1", "r2", "r3"], rows), + # Decile 5 and band z are absent from the support and the rule. + "draw_decile": rng.choice(["1", "2", "3", "4", "5", None], rows), + "band": rng.choice(["x", "y", "z", None], rows), + }, + dtype=object, + ) + expected = _per_row(support, frame, weight_by=RULE, levels=LADDER) + assert set(expected) == {"L0", "L1", "L2", "L3"} + assert _choose(frame, support) == expected + # The answer is per household: it does not depend on the row order. + order = rng.permutation(rows) + assert _choose(frame.iloc[order], support) == [expected[i] for i in order] + + +#: Every level the invented ladders can hold: each set of the two constraints, +#: sampled by the household's own weight or by households. +_RUNGS = tuple( + (constraints, weighted) + for constraints in ((), ("region",), ("draw_decile",), ("region", "draw_decile")) + for weighted in (True, False) +) + + +@st.composite +def _invented(draw): + """An invented support, households and ladder over code domains so small + that empty, absent and missing cells are common: region r9 and decile 9 + are in no support, band z is in no rule, and an input may be missing.""" + areas = draw(st.integers(1, 6)) + + def per_area(elements): + return draw(st.lists(elements, min_size=areas, max_size=areas)) + + weights = {} + for name in ("households", "households_x", "households_y"): + values = per_area(st.integers(0, 2)) + if not any(values): + values[0] = 1 # the decoder requires mass in every weight column + weights[name] = values + support = _conditioned( + region=per_area(st.sampled_from(("r1", "r2"))), + decile=per_area(st.sampled_from(("1", "2"))), + **weights, + ) + rows = draw(st.integers(0, 8)) + + def per_household(codes, stray): + # A code a support or rule holds is twice as likely as a stray one or + # a missing input, so most households find some level. + choices = (*codes, *codes, stray, None) + return draw(st.lists(st.sampled_from(choices), min_size=rows, max_size=rows)) + + households = pd.DataFrame( + { + "key": [f"h{row}" for row in range(rows)], + "lane": ["toy"] * rows, + "region": per_household(("r1", "r2"), "r9"), + "draw_decile": per_household(("1", "2"), "9"), + "band": per_household(("x", "y"), "z"), + }, + dtype=object, + ) + rule = draw( + st.sampled_from((None, RULE, {"input": "band", "map": {"x": "households_y"}})) + ) + rungs = draw( + st.lists( + st.sampled_from([r for r in _RUNGS if rule is not None or not r[1]]), + min_size=1, + max_size=4, + unique=True, + ) + ) + levels = tuple( + FallbackLevel(f"L{index}", constraints, weight_by=weighted) + for index, (constraints, weighted) in enumerate(rungs) + ) + return support, households, rule, levels + + +@settings(max_examples=300, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given(_invented()) +def test_on_invented_supports_the_chooser_equals_the_per_row_loop(invented): + support, households, rule, levels = invented + expected = _per_row(support, households, weight_by=rule, levels=levels) + if expected is None: + with pytest.raises(ValueError) as refused: + _choose(households, support, weight_by=rule, levels=levels) + assert str(refused.value) == NO_LEVEL + else: + assert _choose(households, support, weight_by=rule, levels=levels) == expected + + +def _draw_spec(rule) -> dict: + """The shared draw over an invented support: a constant lane routes every + household, both constraints are optional and one stage samples areas by + ``rule`` (households where it is None or a code is missing).""" + stage = ( + {"level": "area", "weight": "households"} + if rule is None + else {"level": "area", "weight_by": {**rule, "default": "households"}} + ) + return validate_assignment_spec( + { + "version": 1, + "identity": ["key"], + "stream": ["sha256-u53-v1", "toy", 0, 1], + "outputs": { + "area": "area_code", + "system": "area_system", + "basis": "area_basis", + }, + "systems": [ + { + "id": "toy", + "level": "area", + "code_system": "toy", + "vintage": "v1", + "source": "toy_support", + "selector": {"lane": ["toy"]}, + "constraints": [ + {"input": column, "support": matched, "required": False} + for column, matched in CONSTRAINTS.items() + ], + "observed_area": None, + "stages": [stage], + "layers": [], + } + ], + } + ) + + +@settings(max_examples=150, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given(_invented()) +def test_on_invented_supports_the_draw_meets_mass_at_the_chosen_level_only(invented): + """The chosen level is the first at which the shared draw itself finds + mass: drawing there, with every input the level drops missing, lands in + an area that matches the kept inputs and carries the household's weight, + and drawing at any earlier level whose inputs are present is refused for + an empty or absent cell. A household the chooser refuses is refused by + the draw at every level its inputs allow.""" + support, households, rule, levels = invented + spec = _draw_spec(rule) + for row in range(len(households)): + household = households.iloc[[row]] + try: + (chosen,) = _choose(household, support, weight_by=rule, levels=levels) + except ValueError: + chosen = None + for level in levels: + weight = _level_weight(household.iloc[0], level, rule) + if weight is None: + continue + at_level = household.copy() + for column in CONSTRAINTS: + if column not in level.constraints: + at_level[column] = None + if rule is not None and not level.weight_by: + at_level[rule["input"]] = None + if level.name != chosen: + with pytest.raises( + ValueError, match="no common support|zero sampling mass" + ): + assign_atomic(at_level, spec, {"toy": support}) + continue + area = assign_atomic(at_level, spec, {"toy": support})["area_code"].iloc[0] + (index,) = np.flatnonzero(support.arrays["area"] == area) + for column in level.constraints: + assert ( + support.arrays[CONSTRAINTS[column]][index] + == household[column].iloc[0] + ) + assert support.arrays[weight][index] > 0 + break + else: + # A chosen level is one whose inputs the household has. + assert chosen is None + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"weight_by": None}, "no weight_by is given"), + ( + {"levels": (FallbackLevel("L0", ("tenure",), weight_by=False),)}, + "does not map", + ), + ({"constraint_columns": {"region": "households"}}, "support code columns"), + ({"constraint_columns": {"region": "area"}}, "support code columns"), + ( + {"constraint_columns": {"region": "region", "draw_decile": "region"}}, + "support code columns", + ), + ({"default_weight": "decile"}, "default_weight must be"), + ({"weight_by": {"input": "band", "map": {"x": "decile"}}}, "weight_by must"), + ({"weight_by": {**RULE, "default": "households"}}, "weight_by must"), + ({"levels": ()}, "nonempty sequence"), + ( + { + "levels": ( + FallbackLevel("L0", ("region",), weight_by=False), + FallbackLevel("L0", (), weight_by=False), + ) + }, + "names must be distinct", + ), + ], +) +def test_the_chooser_refuses_declarations_it_cannot_read(overrides, message): + with pytest.raises(ValueError, match=message): + _choose(_households(("north", "1", "x")), **overrides) + + +@pytest.mark.parametrize( + ("households", "message"), + [ + (_households(("north", "1", "x")).drop(columns="band"), "lack input columns"), + (_households(("north", 1, "x")), "nonempty strings"), + (_households(("north", " 1", "x")), "nonempty strings"), + (_households(("north", "1", "")), "nonempty strings"), + ], +) +def test_the_chooser_refuses_missing_inputs_and_malformed_codes(households, message): + with pytest.raises(ValueError, match=message): + _choose(households) + + +@pytest.mark.parametrize( + ("name", "constraints", "weight_by", "message"), + [ + ("", ("region",), True, "needs a name"), + ("L0", ["region"], True, "tuple of distinct"), + ("L0", ("region", "region"), True, "tuple of distinct"), + ("L0", ("region",), 1, "is a bool"), + ], +) +def test_a_fallback_level_refuses_a_malformed_declaration( + name, constraints, weight_by, message +): + with pytest.raises(ValueError, match=message): + FallbackLevel(name, constraints, weight_by=weight_by) From e897513e6b5864b4899d4cc5e7df5f9d5d704900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:16:47 +0000 Subject: [PATCH 064/108] Follow up the shared fallback-level chooser's review (#931) choose_fallback_levels now refuses two inputs it cannot honour: - A weight_by rule whose input is also a key of constraint_columns. The shared draw samples by a present code's weight at every level, so a level that kept that input could not sample by the default weight, and a level that dropped it could not keep the household's own weight. The draw could then sample by a weight other than the one the chooser checked and meet a cell without mass. The UK ladder is not affected: its band input is not a constraint. - A households frame that repeats an input column. Such a frame stopped on an uncontrolled numpy error; it is now refused with a constant, count-free text. Tests, on invented supports only: - A missing decile falls to the region levels whether the frame holds it as None, NaN or pd.NA, in object, StringDtype, pandas' str or category columns. Before, only None in object columns was tried. - Two codes that share one weight column both read it, and the Hypothesis rules include such a rule. - Of two identical levels the first is taken, and the Hypothesis ladders may repeat a level. - The default weight is the declared column; each input matches the support column constraint_columns names for it, not a column of its own name; and a missing input never matches a support code spelled "nan", "None" or "". - The two new refusals. Each of the review's mutants that the earlier tests let through (only None read as missing, a missing value read as the text "nan", codes that share a weight column mapped to the wrong column, the last of identical levels winning, a hard-coded default weight, an input's own name read as its support column) fails at least one of these tests; the unchanged code passes. What moves: nothing. The module gains no import and no node hashes it, so no node key moves, and nothing calls the chooser yet. The changelog fragment lists the new refusals and tests, and now says the row-order check uses one shuffled order. Co-Authored-By: Claude Opus 5.5 --- ...931-shared-fallback-level-chooser.added.md | 14 +-- .../build/atomic_support_conditioning.py | 16 ++- .../test_atomic_support_conditioning.py | 101 ++++++++++++++++-- 3 files changed, 114 insertions(+), 17 deletions(-) diff --git a/changelog.d/931-shared-fallback-level-chooser.added.md b/changelog.d/931-shared-fallback-level-chooser.added.md index a9bdcf105c..72bec018fd 100644 --- a/changelog.d/931-shared-fallback-level-chooser.added.md +++ b/changelog.d/931-shared-fallback-level-chooser.added.md @@ -4,11 +4,13 @@ A country-neutral fallback-level chooser for conditioned area draws: `FallbackLe - **The rule.** Each household takes the first level whose inputs are present (every constraint the level keeps and, where it samples by the household's own weight, a code the rule maps) and whose cell, the support's areas matching the household on every kept constraint, has positive summed mass in the weight the household samples by there. It returns each household's level name, in row order. A draw that keeps a household's level's inputs, reads every other input as missing and samples each stage by the same rule then meets no empty cell. The chooser is vectorised, with one grouped mass table per level, read for the households no earlier level took. - **Refusals.** - A household no level takes is refused with one fixed text that names no count, household or value. - - A present input that is not a nonempty string without surrounding whitespace is refused, as the draw refuses it. - - A declaration is refused where a level keeps an input `constraint_columns` does not map, a level samples by the household's own weight but no rule is given, an input maps to the area or to a column that is not a support code column, two inputs map to one column, a weight is not a support weight column, or two levels share a name. + - A present input that is not a nonempty string without surrounding whitespace is refused, as the draw refuses it. A households frame that repeats an input column is refused with a fixed text. + - A declaration is refused where a level keeps an input `constraint_columns` does not map, a level samples by the household's own weight but no rule is given, an input maps to the area or to a column that is not a support code column, two inputs map to one column, a weight is not a support weight column, two levels share a name, or the rule's input is also a key of `constraint_columns`. The draw samples by a present code's weight at every level, so a level could not keep such an input and sample by the default weight, nor drop it and keep the household's own weight. - **No caller yet, and nothing moves.** The chooser names no country, and nothing calls it until the UK geography-first draw. No node binds the module: no transport kernel (the shared kernels New Zealand's graph runs on, the atomic geography and rules kernels among them), no UK graph kernel or gate battery phase, and no transport binding's source closure hashes it, and no US module imports it. So no node key moves. - **Tests.** `tests/engine_free/shared/test_atomic_support_conditioning.py`, on invented supports only: - - a household whose band has mass in its cell takes the first level, one whose cell has households but no banded mass takes the second, and a missing decile falls to the region-only levels; - - a household no level takes is refused with the count-free text; - - on 3,000 invented households the vectorised choice equals a per-row loop over the support's arrays, in any row order; - - two Hypothesis properties on small invented supports and ladders: the chooser equals the per-row loop, and the shared draw itself finds mass at the chosen level and is refused at every earlier level whose inputs are present (and at every level, for a household the chooser refuses). + - a household whose band has mass in its cell takes the first level, one whose cell has households but no banded mass takes the second, and a missing decile falls to the region-only levels, whether the frame holds it as None, NaN or pd.NA, in object, StringDtype, pandas' `str` or category columns; + - two codes that share one weight column both read it, and of two identical levels the first is taken; + - the default weight is the declared column, each input matches the support column `constraint_columns` names for it rather than a column of its own name, and a missing input never matches a support code spelled like a missing value; + - a household no level takes, and a frame that repeats an input column, are refused with count-free texts; + - on 3,000 invented households the vectorised choice equals a per-row loop over the support's arrays, and again under a shuffled row order; + - two Hypothesis properties on small invented supports and ladders, whose rules may share a weight column between codes and whose ladders may repeat a level: the chooser equals the per-row loop, and the shared draw itself finds mass at the chosen level and is refused at every earlier level whose inputs are present (and at every level, for a household the chooser refuses). diff --git a/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py b/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py index 9ce67faaba..147a5d63e0 100644 --- a/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py +++ b/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py @@ -405,7 +405,11 @@ def choose_fallback_levels( ``weight_by`` is true samples by the column ``map`` names for the household's ``input`` code. Every other level samples by ``default_weight``, the draw's ``weight_by`` default or its one - ``weight``. A present input must be a nonempty string without surrounding + ``weight``. The rule's input may not also be a key of + ``constraint_columns``: the draw samples by a present code's weight + whatever the level, so a level could not keep that input without + sampling by the code's weight, nor drop it and keep the household's own + weight. A present input must be a nonempty string without surrounding whitespace, as the draw requires. A draw that keeps a household's level's inputs, reads every other input as missing and samples each stage by this rule then meets no empty cell. @@ -453,6 +457,10 @@ def is_weight(name: object) -> bool: ), "weight_by must map codes of one input to support weight columns", ) + _require( + weight_by is None or weight_by["input"] not in constraint_columns, + "the weight_by input must not also be a constraint input", + ) _require( isinstance(levels, Sequence) and bool(levels) @@ -478,6 +486,12 @@ def is_weight(name: object) -> bool: inputs.add(weight_by["input"]) missing = sorted(inputs - set(households.columns)) _require(not missing, f"households lack input columns {missing}") + _require( + not households.columns[households.columns.isin(sorted(inputs))] + .duplicated() + .any(), + "households repeat an input column", + ) observed = {column: _observed(households[column]) for column in sorted(inputs)} # Each household's own weight, as a position in the rule's weight columns diff --git a/packages/microcosm-build/tests/engine_free/shared/test_atomic_support_conditioning.py b/packages/microcosm-build/tests/engine_free/shared/test_atomic_support_conditioning.py index 44404427ad..2e1f757821 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_atomic_support_conditioning.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_atomic_support_conditioning.py @@ -326,6 +326,8 @@ def test_the_shared_operator_draws_on_the_added_columns(): #: Each household input to the support column it must match. CONSTRAINTS = {"region": "region", "draw_decile": "decile"} RULE = {"input": "band", "map": {"x": "households_x", "y": "households_y"}} +#: Two codes that share one weight column. +SHARED_RULE = {"input": "band", "map": {"x": "households_y", "y": "households_y"}} NO_LEVEL = ( "Atomic support conditioning: a household has no fallback level whose " "inputs are present and whose cell has sampling mass" @@ -441,13 +443,21 @@ def test_a_household_whose_cell_has_only_households_takes_level_one(): assert _choose(rows) == ["L1", "L1"] -def test_a_missing_decile_falls_to_the_region_levels(): +@pytest.mark.parametrize("missing", [None, np.nan, pd.NA], ids=["None", "nan", "NA"]) +@pytest.mark.parametrize( + "dtype", + [object, "string", "str", "category"], + ids=["object", "string", "str", "category"], +) +def test_a_missing_decile_falls_to_the_region_levels(missing, dtype): + """A missing input reads as missing whichever way a frame holds it: None, + NaN or pd.NA, in object, StringDtype, pandas' str or category columns.""" rows = _households( - ("north", None, "x"), # band x has mass in the north: L2 - ("south", None, "x"), # but none in the south: L3 - ("north", None, None), # no band either: L3 - ("north", "1", None), # a decile and no band: L1 - ) + ("north", missing, "x"), # band x has mass in the north: L2 + ("south", missing, "x"), # but none in the south: L3 + ("north", missing, missing), # no band either: L3 + ("north", "1", missing), # a decile and no band: L1 + ).astype(dtype) assert _choose(rows) == ["L2", "L3", "L3", "L1"] @@ -469,6 +479,61 @@ def test_without_households_the_chooser_returns_no_level(): assert _choose(_households()) == [] +def test_two_codes_sharing_a_weight_column_both_read_it(): + # households_y has mass only in a2 (south, 1). + rows = _households(("south", "1", "x"), ("south", "1", "y"), ("north", "1", "y")) + assert _choose(rows, weight_by=SHARED_RULE) == ["L0", "L0", "L1"] + + +def test_identical_levels_take_the_first_name(): + levels = ( + FallbackLevel("first", ("region",), weight_by=False), + FallbackLevel("second", ("region",), weight_by=False), + ) + assert _choose(_households(("north", "1", "x")), levels=levels) == ["first"] + + +@pytest.mark.parametrize("spelled", ["nan", "None", ""]) +def test_a_missing_input_never_matches_a_code_spelled_like_missing(spelled): + support = _conditioned( + region=("north", "north"), decile=(spelled, "2"), households=(5, 0) + ) + rows = _households( + ("north", None, None), ("north", np.nan, None), ("north", pd.NA, None) + ) + levels = ( + FallbackLevel("decile", ("region", "draw_decile"), weight_by=False), + FallbackLevel("region", ("region",), weight_by=False), + ) + assert _choose(rows, support, weight_by=None, levels=levels) == ["region"] * 3 + + +def test_the_default_weight_is_the_declared_column(): + # The south has dwellings but no households. + support = _conditioned( + region=("north", "south"), + decile=("1", "1"), + households=(4, 0), + dwellings=(0, 3), + ) + levels = (FallbackLevel("region", ("region",), weight_by=False),) + rows = _households(("south", "1", None)) + chosen = _choose( + rows, support, weight_by=None, default_weight="dwellings", levels=levels + ) + assert chosen == ["region"] + + +def test_inputs_follow_constraint_columns_not_their_own_names(): + # The input "region" matches the support's decile column and "draw_decile" + # its region column: (decile 1, north) is a0, which has households. + swapped = {"region": "decile", "draw_decile": "region"} + levels = (FallbackLevel("both", ("region", "draw_decile"), weight_by=False),) + rows = _households(("1", "north", None)) + chosen = _choose(rows, weight_by=None, levels=levels, constraint_columns=swapped) + assert chosen == ["both"] + + def test_the_vectorised_choice_equals_a_per_row_loop(): rng = np.random.default_rng(931) areas, rows = 60, 3000 @@ -511,6 +576,9 @@ def test_the_vectorised_choice_equals_a_per_row_loop(): for constraints in ((), ("region",), ("draw_decile",), ("region", "draw_decile")) for weighted in (True, False) ) +#: The rules besides RULE an invented ladder may sample by: one that maps a +#: single code, and one whose two codes share a weight column. +_RULES = ({"input": "band", "map": {"x": "households_y"}}, SHARED_RULE) @st.composite @@ -552,15 +620,13 @@ def per_household(codes, stray): }, dtype=object, ) - rule = draw( - st.sampled_from((None, RULE, {"input": "band", "map": {"x": "households_y"}})) - ) + rule = draw(st.sampled_from((None, RULE, *_RULES))) + # A rung may repeat, so a tie between identical levels goes to the first. rungs = draw( st.lists( st.sampled_from([r for r in _RUNGS if rule is not None or not r[1]]), min_size=1, max_size=4, - unique=True, ) ) levels = tuple( @@ -687,6 +753,11 @@ def test_on_invented_supports_the_draw_meets_mass_at_the_chosen_level_only(inven ({"default_weight": "decile"}, "default_weight must be"), ({"weight_by": {"input": "band", "map": {"x": "decile"}}}, "weight_by must"), ({"weight_by": {**RULE, "default": "households"}}, "weight_by must"), + # The draw would read the region for its weight at every level. + ( + {"weight_by": {"input": "region", "map": {"north": "households_x"}}}, + "must not also be a constraint input", + ), ({"levels": ()}, "nonempty sequence"), ( { @@ -718,6 +789,16 @@ def test_the_chooser_refuses_missing_inputs_and_malformed_codes(households, mess _choose(households) +def test_a_repeated_input_column_is_refused_naming_no_count(): + rows = _households(("north", "1", "x"), ("south", "1", "y")) + rows = pd.concat([rows, rows[["draw_decile"]]], axis=1) + with pytest.raises(ValueError) as refused: + _choose(rows) + assert str(refused.value) == ( + "Atomic support conditioning: households repeat an input column" + ) + + @pytest.mark.parametrize( ("name", "constraints", "weight_by", "message"), [ From 819197209a75349660e79d12ed9c31a5c9f053c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:05:27 +0000 Subject: [PATCH 065/108] Add the spine's area, local-authority, BRMA, fuel-relief, SPI-copy area and private-rent transforms, unwired (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three new UK modules hold the code for the clone-first spine's geography and private-rent stages. No stage declares them and no node binds them yet. Each transform checks the stage it runs as against the reviewed declaration its module states, which the geography declarations will carry. - atomic_area_conditioning: the conditioned area draw. Its definition over the three conditioned supports, held to canonical bytes, keeps the FRS region as a required constraint and the IMD decile as an optional one. England and Wales and Scotland sample by the area's households in the household's observed council-tax band, or by all its households where none is observed; Northern Ireland samples by households. The draw uses its own stream with seed 0. The fallback ladder runs from L0 (decile and band) to L3 (region and households). The draw reads the decile as "1" to "10" and the band only where the FRS observed it; rows the SPI support channel copied read neither. - spine_geography: G2 (atomic_area_assign) draws, derives and validates each household's area, and a failed validation refuses the stage. G3 (local_authority) resolves the engine's local authority through the names roster. G4 (area_brma) reads Great Britain's BRMA off the drawn area through the area-to-BRMA layer, and draws Northern Ireland's with frs_brma's keys, seed and salts. G5 (rfdr) sets the fuel-relief flag from the drawn output area and holds the RFDR layer to the supports' areas. spi_copy_areas gives every SPI copy and income band donor its own geography key, an area drawn at L3, and its own local authority, BRMA and flag. - spine_private_rent: G7 (private_rent) re-draws each Great Britain private renter's rent off its BRMA's list of rents. Every row is its own spine household, tie-breaks use the stage's own stream, and the sampled rung's region rule counts true sources. Evidence counts distinct true source households through rounded_count, never per area or per cell, and prints no key or code of a household. G2, G3 and G4 count over every clone; G5, the copy stage and G7 count among clone-0 rows. Each stage's evidence is byte-equal at K = 1 and K = 3 but for the clone count. The copy stage's evidence says its validation and roster check did not run on a frame without a copy. frs_brma gains draw_household_brmas, which draws any subset of households with today's keys; add_frs_brma now calls it. area_flags in rfdr_area_layer gains a roster. Why. In the plan's target order, atomic_area_assign, local_authority and rfdr run right after geographic_identity, ahead of frs_take_up. area_brma takes frs_brma's slot, since LHA_category reads would_claim_uc, which frs_take_up draws (C-1). spi_copy_areas follows spi_income_band_donors, and private_rent follows spi_housing_shell. The code comes first; a later commit of the group declares the stages and regenerates H2. María's rulings of 6 Nov: - Area columns ship on both lines (item 4), so the spine writes them. - SPI copies and band donors draw their own area within their region by census households, without the decile, and the local-authority, BRMA and fuel-relief stages re-run on them (item 7). - Northern Ireland keeps the decile constraint (item 9); the measurement tool checks it later. - Where a fit folds clones that differ, it keeps the lowest-id record (item 10), clone 0's. So every draw here is keyed per clone, and clone 0's is the same at every clone count. As María ruled on 11 Nov (C-9), Northern Ireland's SPI-channel copies re-draw their BRMA. One engine call over those copies' households alone materialises their LHA_category, which reads only the household's own bedroom entitlement and shared-rate status. Then frs_brma's count-table draw runs on the copies' own benefit units with area_brma's seed and salts. The call's scratch frame carries no clone index, so the UK adapter runs it as one simulation over every clone's copies, as at one clone, rather than refusing a call over part of the spine's pool, which a clone without a Northern Ireland copy would otherwise make it. The reviewed operation and the evidence declare that one simulation (a default, listed for María and Max). What moves. No value. On the synthetic H2 fixture, the legacy oracle's final frame (every table, column and dtype, the weights, the strata and the mass log), its stage records and every stage's evidence digest are identical before and after this commit, and the fixture regenerates byte-identically. frs_brma's refusals no longer print household or benefit-unit ids. area_flags' malformed-code refusal no longer prints a count or a code, and its unknown-system refusal names the accepted systems, not the frame's values. Keys move for the stage kernels that bind frs_brma (frs_brma, was_wealth, was_lisa, uc_take_up, uc_deduction_attributes and childcare_copy_take_up) and for the dense line's uk.full.geography.brma and uk.full.geography.rfdr nodes (the layer modules' edits), with every node downstream of them. No transport (NZ, BE, AM), US or battery-phase key moves, and no node hashes the three new modules. No declaration, generated file or gates change. Tests. The fixtures are toy conditioned supports whose public area codes resolve against the committed names roster, BRMA layer, rent lists, constituency knots and RFDR layer; invented households cloned and keyed by the spine's own steps; a recording stand-in for the rules engine; and a stand-in for spi_support_channel and spi_income_band_donors. The tests hold: - each household's level and region, the decile read as "1" to "10", and the chooser called with the definition's own rule, default and ladder; - clone 0's areas at K = 1, 2 and 3, and its copy keys, areas and BRMAs at K = 2 and 3, equal to K = 1's; - row-order independence, the validation (failing the stage on a ladder the mapping does not give) and the written dtypes; - Great Britain's BRMAs against the layer and Northern Ireland's against frs_brma; - the copy stage's own keys and its Northern Ireland draws, and its engine call as one simulation under the UK adapter's clone rule at K = 2 and 3 with Northern Ireland copies in clone 0 alone; - G7 against localise_rents on a copy-free frame; - count-free refusals, the frame's before any engine call; - every evidence count byte-equal at K = 1 and K = 3. test_uk_frs_stochastic holds add_frs_brma to the draws it made before, writes its household pick out, and holds a household subset to the whole frame's draws. A mounted test, skipped without the supports, holds the chooser to the committed zero-mass tables. Co-Authored-By: Claude Opus 5.5 --- .../931-spine-geography-transforms.added.md | 32 + .../uk_runtime/atomic_area_conditioning.py | 672 ++++++ .../build/uk_runtime/brma_area_layer.py | 12 +- .../microcosm/build/uk_runtime/frs_brma.py | 102 +- .../build/uk_runtime/rfdr_area_layer.py | 69 +- .../build/uk_runtime/spine_geography.py | 2043 +++++++++++++++++ .../build/uk_runtime/spine_private_rent.py | 631 +++++ .../uk/test_uk_atomic_area_conditioning.py | 566 +++++ .../uk/test_uk_conditioned_area_supports.py | 71 + .../engine_free/uk/test_uk_frs_stochastic.py | 194 ++ .../engine_free/uk/test_uk_rfdr_area_layer.py | 70 +- .../engine_free/uk/test_uk_spine_geography.py | 1867 +++++++++++++++ .../uk/test_uk_spine_private_rent.py | 993 ++++++++ .../uk_conditioned_support_fixtures.py | 472 ++++ 14 files changed, 7746 insertions(+), 48 deletions(-) create mode 100644 changelog.d/931-spine-geography-transforms.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_conditioning.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/spine_geography.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/spine_private_rent.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_atomic_area_conditioning.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_spine_geography.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_spine_private_rent.py create mode 100644 test_support/microcosm_build/uk_conditioned_support_fixtures.py diff --git a/changelog.d/931-spine-geography-transforms.added.md b/changelog.d/931-spine-geography-transforms.added.md new file mode 100644 index 0000000000..4b7ff84b60 --- /dev/null +++ b/changelog.d/931-spine-geography-transforms.added.md @@ -0,0 +1,32 @@ +The clone-first UK spine's area, local-authority, BRMA, fuel-relief, SPI-copy area and private-rent transforms (#931), unwired: no stage declares them and no node binds the three new modules, so no output moves. Each transform checks the stage it runs as against the reviewed declaration its module states (name, household grain, outputs and rewrites, the reviewed operations verbatim and its artifacts, each with `runtime_sha256_required`), which the geography declarations will carry. + +- **The conditioned area draw** (`microcosm.build.uk_runtime.atomic_area_conditioning`). + - The definition over the three conditioned supports, held to canonical bytes, keeps the FRS region as a required constraint and the IMD decile as an optional one. England and Wales and Scotland sample each household by the area's households in its observed council-tax band, or by all the area's households where it has none; Northern Ireland, which has domestic rates, samples by households. The layers are the ten ladder columns, and the stream is `uk-spine-atomic-area-v1` with seed 0. + - The fallback ladder runs from L0 (decile and band) to L3 (region and households); Northern Ireland takes L1 and L3. + - The draw reads the decile as "1" to "10" (missing for 0) and the band only where `council_tax_band_observed` is true. A row the SPI support channel copied reads neither, so it draws at L3. + - The shared chooser picks each household's level with the definition's own rule and default. +- **The area stages** (`microcosm.build.uk_runtime.spine_geography`). + - G2, `UKAtomicAreaAssignStageTransform` (`atomic_area_assign`), draws, derives and validates every household's area on one narrow frame and writes the three assignment and ten ladder columns as `StringDtype("python")`. A validation that fails refuses the stage. + - G3, `UKLocalAuthorityStageTransform` (`local_authority`), resolves `local_authority` from `local_authority_code` through the committed names roster, never defaulted, as `StringDtype("python")`. + - G4, `UKAreaBRMAStageTransform` (`area_brma`, to take `frs_brma`'s slot), makes one engine call over the whole frame for `LHA_category`. Great Britain reads `brma` off the drawn LSOA (England and Wales) or output area (Scotland) through the committed area-to-BRMA layer. Northern Ireland draws from the census counts with `frs_brma`'s keys, seed and salts, so its values are `frs_brma`'s. `brma` stays an object column, and a frame G4 cannot place is refused before the engine call. + - G5, `UKRFDRStageTransform` (`rfdr`), sets `in_rural_fuel_duty_relief_area` from the drawn output area through the committed RFDR layer. It refuses a layer that flags an output area its system's support does not hold, and reports a system with no flagged household instead of refusing. + - `UKSPICopyAreasStageTransform` (`spi_copy_areas`, to run after `spi_income_band_donors`) gives every SPI copy and income band donor a geography key over its own copy path, an area drawn at L3 with G2's definition, and then its own local authority, BRMA and flag. As María ruled on 11 Nov (C-9), Northern Ireland's copies draw their BRMA from the census counts on their own benefit units, with G4's seed and salts, after one engine call over their households alone. That call carries no clone index, so it is one simulation over every clone's copies, as at one clone: the UK adapter refuses a call over part of its pool, which a clone without a Northern Ireland copy would otherwise make it. Great Britain copies' areas are checked against the layer before the call. Each rewrite keeps its column's dtype, and every other row is left as it was. +- **Private rents** (`microcosm.build.uk_runtime.spine_private_rent`). G7, `UKPrivateRentStageTransform` (`private_rent`, to run after `spi_housing_shell`), re-draws each Great Britain private renter's rent with `private_rent_localisation.localise_rents`, every row its own spine household keyed on its `geography_household_key`. Tie-breaks are keyed on `(key, "rent_rank")` on the stream `("sha256-u53-v1", "uk-spine-private-rent-v1", 0, 0)`, so tied renters and the clones of one renter take ranks of their own and clone 0's tie-breaks are the same at every K. The sampled rung's thin-region rule counts true source households, and the stage is strict unless the spine is sampled. Only `rent` is rewritten: reported Housing Benefit, Universal Credit and mortgages, the weights and every other column are unchanged, and the stage records its household mass receipt. +- **Evidence.** Every count is distinct true source households through `rounded_count`, never per area or per cell, and no key or code of a household is printed. G2 (by system, level and fallback reason), G3 and G4 (by grain, and Northern Ireland's sources drawn) count over every clone. G5's flagged sources, the copy stage's copies, donors and flagged copies, and G7's re-drawn sources count among clone-0 rows; G7 also gives the cells ranked against their region or against Great Britain. The copy stage's evidence says its validation and roster check did not run on a frame without a copy. +- **Changed modules.** + - `frs_brma.draw_household_brmas` draws the BRMAs of any subset of households with today's keys; `add_frs_brma` calls it and its values are unchanged. Its refusals no longer print household or benefit-unit ids. + - `rfdr_area_layer.area_flags` gains `roster`. Its malformed-code refusal no longer prints a count or a code, and its unknown-system refusal names the accepted systems instead of the frame's values. + - The BRMA and RFDR layer docstrings describe the spine's stages. +- **Keys.** The stage kernels that bind `frs_brma` (`frs_brma`, `was_wealth`, `was_lisa`, `uc_take_up`, `uc_deduction_attributes` and `childcare_copy_take_up`) re-key with it, and so do the dense line's geography BRMA and RFDR nodes (`uk.full.geography.brma@1` and `uk.full.geography.rfdr@1`, after the layer modules' edits), with every node downstream of them. No transport key (NZ, BE, AM), US key or battery phase moves, and no node hashes the three new modules. No declaration, generated file or `gates.json` changes. +- **Values.** None moves. On the synthetic H2 fixture, the legacy oracle's final frame (every table, column and dtype, the weights, the strata and the mass log), its stage records and every stage's evidence digest are identical before and after this change, and the fixture regenerates byte-identically. +- **Tests**, on toy conditioned supports whose public area codes resolve against the committed resources (`test_support/microcosm_build/uk_conditioned_support_fixtures.py`), invented households, a recording stand-in for the rules engine and a stand-in for the support channel's two stages: + - each household's level and region, the decile read as "1" to "10", and the chooser called with the definition's own rule, default and ladder; + - clone 0's areas at K = 1, 2 and 3, and its copy keys, areas and BRMAs at K = 2 and 3, equal to K = 1's, and row-order independence; + - G2's and the copy stage's validation refusing a ladder the mapping does not give, and the written dtypes; + - Great Britain's BRMAs against the layer, Northern Ireland's against `frs_brma`, and `frs_brma`'s household pick written out; + - the copy stage's own keys and Northern Ireland draws, and its engine call as one simulation under the UK adapter's clone rule at K = 2 and 3 with Northern Ireland copies in clone 0 alone; + - G7 against `localise_rents` on a copy-free frame; + - count-free refusals, the frame's before any engine call; + - every evidence count byte-equal at K = 1 and K = 3. + + A mounted test, skipped without the supports, holds the chooser to the committed zero-mass tables. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_conditioning.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_conditioning.py new file mode 100644 index 0000000000..ab9895e371 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_conditioning.py @@ -0,0 +1,672 @@ +"""The UK's conditioned area draw: its definition, inputs and fallback ladder (microcosm#931). + +G2 of the clone-first plan draws each household's atomic area within its FRS +region, conditioned on what the FRS records about the real address (O14 iv +and v): the address's IMD decile within its nation, and the household's +council-tax band where the survey gives it a band its region uses. The shared +operator takes both (:mod:`microcosm.build.atomic_geography`: a code +constraint that may be missing, and ``weight_by``, a weight column chosen per +household), and the shared chooser +(:func:`~microcosm.build.atomic_support_conditioning.choose_fallback_levels`) +picks the level each household is drawn at. This module declares the UK's use +of both; it runs no stage: + +- :func:`uk_conditioned_assignment_definition` is the operator's declaration + over the three conditioned supports. It is today's region-only declaration + (``atomic_area_support.uk_atomic_assignment_definition``) with the + conditioned sources; the region (required) and the transient decile input + (``required: false``) as constraints; England and Wales and Scotland + sampling by the household's band weight, or by households where it has no + band; Northern Ireland, which has domestic rates, by households; the ten + ladder columns as its only layers; and the spine's own stream. +- :data:`UK_ATOMIC_FALLBACK_LEVELS` is the ladder, in the plan's order: L0 + the decile and the band's weight, L1 the decile and households, L2 the + region and the band's weight, L3 the region and households. Northern + Ireland, with no band weights, takes L1 and L3 only. +- :func:`uk_draw_inputs` is the narrow transient frame the draw reads: each + household's geography key and region, its decile as "1" to "10" (missing + for 0, no observed decile) and its band where ``council_tax_band_observed`` + (missing otherwise). Rows the SPI support channel copied get both inputs + missing, so they draw at L3, the level the SPI-copy area stage draws them + at, though a copy keeps its parent's decile and band flag (C-5, C-10). +- :func:`uk_draw_levels` gives each household its level: the first whose + inputs are present and whose cell has mass in the weight it samples by + there, the chooser's rule, with the chooser's arguments read from the + definition itself, so the chooser and the draw agree. +- :func:`uk_draw_frame` sets every input a household's level drops to + missing, so one definition serves every level. +- :func:`uk_fallback_reasons` says, for the evidence, why each household is + drawn where it is. + +The module is its own, not part of :mod:`atomic_area_support`, whose hash the +national problem, the dense measures, the inspection export and the geography +identity bind: it imports that module's constants and two of its helpers and +leaves it as it is (C-28). + +Refusals give columns and declared constants only, never a key, an id, a code +of a household or a count. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from numbers import Integral +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.atomic_geography import ( + AtomicSupport, + decode_atomic_support, + validate_assignment_spec, +) +from microcosm.build.atomic_support_conditioning import ( + FallbackLevel, + choose_fallback_levels, +) +from microcosm.build.source_lineage import copy_paths +from microcosm.build.uk_runtime.atomic_area_support import ( + ASSIGNMENT_COLUMNS, + BAND_WEIGHT_COLUMNS, + CONDITIONED_SOURCES, + IMD_DECILE_COLUMN, + IMD_DECILES, + SYSTEMS, + _profile, + uk_conditioned_support_base, +) +from microcosm.build.uk_runtime.council_tax_constants import UK_COUNCIL_TAX_BANDS +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, +) +from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE + +__all__ = [ + "BAND_INPUT_COLUMN", + "COUNCIL_TAX_BAND_COLUMN", + "COUNCIL_TAX_BAND_OBSERVED_COLUMN", + "DECILE_INPUT_COLUMN", + "DEFAULT_WEIGHT_COLUMN", + "DRAW_INPUT_COLUMNS", + "IDENTITY_COLUMN", + "REGION_COLUMN", + "SUPPORT_CHANNEL_BRANCH", + "UK_ATOMIC_FALLBACK_LEVELS", + "UK_DRAW_CONSTRAINTS", + "UK_FALLBACK_STATUSES", + "UK_SPINE_AREA_COLUMNS", + "UK_SPINE_ATOMIC_AREA_SEED", + "UK_SPINE_ATOMIC_AREA_STREAM", + "UK_SPINE_LADDER_COLUMNS", + "UK_SPINE_LAYERS", + "UNOBSERVED_IMD_DECILE", + "uk_band_weight_map", + "uk_conditioned_assignment_definition", + "uk_conditioned_supports", + "uk_draw_frame", + "uk_draw_inputs", + "uk_draw_levels", + "uk_draw_systems", + "uk_fallback_reasons", + "uk_support_channel_copies", + "uk_system_fallback_levels", +] + +#: The key every draw is keyed on: each clone's geography key, which the +#: spine's geographic_identity stage writes and the SPI-copy area stage +#: rewrites on copies (named in the geographic_clones leaf; +#: ``atomic_area_support.IDENTITY_COLUMN`` is the same name). +IDENTITY_COLUMN = GEOGRAPHY_HOUSEHOLD_KEY_COLUMN +#: The spine's own stream for the area draw, and its seed: 0, declared by the +#: stage (C-3), one constant for both lines, so clone 0 nests at every K. +UK_SPINE_ATOMIC_AREA_STREAM = "uk-spine-atomic-area-v1" +UK_SPINE_ATOMIC_AREA_SEED = 0 +_STREAM_ALGORITHM = "sha256-u53-v1" + +#: The household columns the inputs are read from. The band flag and the +#: unobserved decile are frs_spine's (``FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN`` +#: and ``FRS_IMD_DECILE_UNOBSERVED``), spelled here so that an edit to that +#: module does not re-key the area stages; a test holds the spellings equal. +REGION_COLUMN = "region" +COUNCIL_TAX_BAND_COLUMN = "council_tax_band" +COUNCIL_TAX_BAND_OBSERVED_COLUMN = "council_tax_band_observed" +UNOBSERVED_IMD_DECILE = 0 +#: The transient inputs the draw reads: never stage outputs, they live on the +#: narrow frame :func:`uk_draw_inputs` builds and go with it. +DECILE_INPUT_COLUMN = "atomic_draw_imd_decile" +BAND_INPUT_COLUMN = "atomic_draw_council_tax_band" +DRAW_INPUT_COLUMNS = ( + IDENTITY_COLUMN, + REGION_COLUMN, + DECILE_INPUT_COLUMN, + BAND_INPUT_COLUMN, +) +#: Each constraint input and the support code column it must match. +UK_DRAW_CONSTRAINTS: Mapping[str, str] = MappingProxyType( + {REGION_COLUMN: "frs_region", DECILE_INPUT_COLUMN: IMD_DECILE_COLUMN} +) +#: The weight a household samples by where it has no band, or its level +#: drops it. +DEFAULT_WEIGHT_COLUMN = "households" +#: The lineage branch whose copies draw without inputs: the SPI support +#: channel, which copies households and makes the income band donors. +SUPPORT_CHANNEL_BRANCH = "spi_support_channel" + +#: The ten ladder code columns the draw writes: the geography ladder's columns +#: without the engine's ``local_authority``, which the local-authority stage +#: resolves from ``local_authority_code``. Spelled here, as +#: ``geography_ladder.UK_GEOGRAPHY_LADDER_COLUMNS`` orders them, so that an +#: edit to that module does not re-key the area stages; a test holds them +#: equal. +UK_SPINE_LADDER_COLUMNS = ( + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code", +) +#: Each layer's support column and output: the atomic area itself for +#: ``oa_code``, the column of the same name otherwise. The nation-native +#: aliases (``atomic_area_support.UK_NATIVE_ALIAS_COLUMNS``) are left out: +#: they are missing outside their own nation and dropped at the export +#: boundary. +UK_SPINE_LAYERS = tuple( + ("area" if column == "oa_code" else column, column) + for column in UK_SPINE_LADDER_COLUMNS +) +#: Every column the draw writes: the three assignment columns, then the ladder. +UK_SPINE_AREA_COLUMNS = (*ASSIGNMENT_COLUMNS, *UK_SPINE_LADDER_COLUMNS) + +#: The fallback ladder, in the plan's order. A household takes the first level +#: whose inputs are present and whose cell has mass in the weight it samples +#: by there; a level reads every input it does not keep as missing. +UK_ATOMIC_FALLBACK_LEVELS = ( + FallbackLevel("L0", (REGION_COLUMN, DECILE_INPUT_COLUMN), True), + FallbackLevel("L1", (REGION_COLUMN, DECILE_INPUT_COLUMN), False), + FallbackLevel("L2", (REGION_COLUMN,), True), + FallbackLevel("L3", (REGION_COLUMN,), False), +) +_LEVELS = {level.name: level for level in UK_ATOMIC_FALLBACK_LEVELS} +_BAND_WEIGHT_PREFIX = "households_band_" +#: The statuses :func:`uk_fallback_reasons` gives each input, in order. +UK_FALLBACK_STATUSES: Mapping[str, tuple[str, ...]] = MappingProxyType( + { + "decile": ("kept", "unobserved", "dropped"), + "band": ("kept", "unobserved", "dropped", "none"), + } +) + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("UK conditioned area draw: " + message) + + +def uk_band_weight_map(system: str) -> dict[str, str]: + """Each council-tax band the system's support weighs, and its column. + + ``{"A": "households_band_a", ...}``: A to I in England and Wales, A to H + in Scotland, none in Northern Ireland (``BAND_WEIGHT_COLUMNS``). + """ + + _require(system in SYSTEMS, "unknown area system") + columns = BAND_WEIGHT_COLUMNS[system] + mapping = { + column.removeprefix(_BAND_WEIGHT_PREFIX).upper(): column for column in columns + } + _require( + all(column.startswith(_BAND_WEIGHT_PREFIX) for column in columns) + and set(mapping) <= set(UK_COUNCIL_TAX_BANDS), + "band weight columns must name council-tax bands", + ) + return mapping + + +def uk_system_fallback_levels(system: str) -> tuple[FallbackLevel, ...]: + """The levels a system's households take: all four where the support + weighs bands, L1 and L3 in Northern Ireland, which has none.""" + + _require(system in SYSTEMS, "unknown area system") + if BAND_WEIGHT_COLUMNS[system]: + return UK_ATOMIC_FALLBACK_LEVELS + return tuple(level for level in UK_ATOMIC_FALLBACK_LEVELS if not level.weight_by) + + +def _seed(seed: object) -> int: + _require( + isinstance(seed, Integral) + and not isinstance(seed, (bool, np.bool_)) + and 0 <= int(seed) < 2**32, + "seed must be a bounded integer", + ) + return int(seed) + + +def uk_conditioned_supports(supports: Mapping[str, bytes]) -> dict[str, AtomicSupport]: + """The three conditioned supports, decoded, each held to canonical bytes. + + Each payload must be exactly what + ``atomic_area_support.assemble_uk_conditioned_area_support`` makes of its + system's canonical support, as the definition requires. + """ + + _require( + isinstance(supports, Mapping) and set(supports) == set(SYSTEMS), + "the three systems' supports are required", + ) + decoded = {} + for system in SYSTEMS: + payload = supports[system] + _require(type(payload) is bytes, "immutable support bytes required") + # Refuses bytes that are not the canonical conditioned output. + uk_conditioned_support_base(payload) + support = decode_atomic_support(payload) + _require(support.metadata["system"] == system, "support system differs") + decoded[system] = support + return decoded + + +def uk_conditioned_assignment_definition( + supports: Mapping[str, bytes], + *, + seed: int, +) -> dict: + """Declare the spine's conditioned area draw over the three supports. + + As ``atomic_area_support.uk_atomic_assignment_definition`` declares the + region-only draw, but over the conditioned supports + (``CONDITIONED_SOURCES``, canonical bytes only), with the region + (required) and :data:`DECILE_INPUT_COLUMN` (``required: false``) as + constraints, one stage at the atomic area that samples by + ``weight_by`` (input :data:`BAND_INPUT_COLUMN`, each band to its + ``households_band_*`` column, default ``households``) where the system + weighs bands and by ``households`` in Northern Ireland, the ten ladder + columns as layers (:data:`UK_SPINE_LAYERS`), and the stream + ``("sha256-u53-v1", "uk-spine-atomic-area-v1", 0, seed)``. + + A missing input is skipped (the decile) or samples by the default weight + (the band), so one definition draws every level of the ladder: a level's + draw sets the inputs it drops to missing (:func:`uk_draw_frame`). + """ + + seed = _seed(seed) + decoded = uk_conditioned_supports(supports) + systems = [] + for system in SYSTEMS: + support = decoded[system] + level, vintage, regions, _ = _profile(system) + _require( + all( + support.metadata[key] == value + for key, value in ( + ("system", system), + ("level", level), + ("code_system", "uk_gss"), + ("vintage", vintage), + ) + ), + "support system identity differs", + ) + bands = uk_band_weight_map(system) + stage = ( + { + "level": "area", + "weight_by": { + "input": BAND_INPUT_COLUMN, + "map": bands, + "default": DEFAULT_WEIGHT_COLUMN, + }, + } + if bands + else {"level": "area", "weight": DEFAULT_WEIGHT_COLUMN} + ) + columns = support.metadata["columns"] + systems.append( + { + "id": system, + "level": level, + "code_system": "uk_gss", + "vintage": vintage, + "source": CONDITIONED_SOURCES[system], + "selector": {REGION_COLUMN: list(regions)}, + "constraints": [ + { + "input": column, + "support": matched, + "required": column == REGION_COLUMN, + } + for column, matched in UK_DRAW_CONSTRAINTS.items() + ], + "observed_area": None, + "stages": [stage], + "layers": [ + { + "input": column, + "output": output, + **{ + key: columns[column][key] + for key in ("source", "vintage", "relation") + }, + } + for column, output in UK_SPINE_LAYERS + ], + } + ) + return validate_assignment_spec( + { + "version": 1, + "identity": [IDENTITY_COLUMN], + "stream": [_STREAM_ALGORITHM, UK_SPINE_ATOMIC_AREA_STREAM, 0, seed], + "outputs": dict( + zip(("area", "system", "basis"), ASSIGNMENT_COLUMNS, strict=True) + ), + "systems": systems, + } + ) + + +def uk_support_channel_copies(household: pd.DataFrame) -> np.ndarray: + """Whether each row is a copy the SPI support channel made. + + Read from the lineage columns through the shared reader on the UK's copy + lineage, never from ids or values: a row whose copy path takes the + support channel's branch, an SPI copy or an income band donor or any copy + a later stage makes of one. A table without the channel's columns has no + such row. + """ + + branch = next( + b for b in UK_COPY_LINEAGE.branches if b.name == SUPPORT_CHANNEL_BRANCH + ) + if not any(column in household.columns for column in branch.columns): + return np.zeros(len(household), dtype=bool) + _, paths = copy_paths(household, UK_COPY_LINEAGE, require=(SUPPORT_CHANNEL_BRANCH,)) + return np.fromiter( + (any(step == SUPPORT_CHANNEL_BRANCH for step, _ in path) for path in paths), + dtype=bool, + count=len(paths), + ) + + +def _strings(series: pd.Series, column: str) -> list[str]: + values = series.to_numpy(dtype=object) + _require( + not series.isna().any() + and all( + isinstance(value, str) and value and value.strip() == value + for value in values + ), + f"{column} must hold a nonempty code on every row", + ) + return [str(value) for value in values] + + +def uk_draw_inputs(household: pd.DataFrame) -> pd.DataFrame: + """The narrow transient frame the area draw reads, one row per household. + + Columns :data:`DRAW_INPUT_COLUMNS`, in the household table's row order + and index, each in the graph's string dtype (``StringDtype("python")``): + + - the geography key and the region, as the table holds them; + - :data:`DECILE_INPUT_COLUMN`: the household's ``imd_decile`` as "1" to + "10", missing where it is 0, no observed decile (O14 v); + - :data:`BAND_INPUT_COLUMN`: its ``council_tax_band`` where + ``council_tax_band_observed`` is true, missing otherwise (O14 iv). + + A row the SPI support channel copied (:func:`uk_support_channel_copies`) + gets both inputs missing, whatever its decile and flag, which it keeps + from its parent (C-5, C-10). + + Refused, by column: a missing column, a missing or blank key or region, a + decile column that is not integer or holds codes outside 0 to 10, a flag + column that is not a complete boolean, and an observed band that is not a + band A to I. + """ + + _require(isinstance(household, pd.DataFrame), "the household table is a DataFrame") + required = ( + IDENTITY_COLUMN, + REGION_COLUMN, + IMD_DECILE_COLUMN, + COUNCIL_TAX_BAND_COLUMN, + COUNCIL_TAX_BAND_OBSERVED_COLUMN, + ) + missing = [column for column in required if column not in household.columns] + _require(not missing, f"the household table lacks {missing}") + keys = _strings(household[IDENTITY_COLUMN], IDENTITY_COLUMN) + regions = _strings(household[REGION_COLUMN], REGION_COLUMN) + decile = household[IMD_DECILE_COLUMN] + _require( + pd.api.types.is_integer_dtype(decile.dtype) + and not pd.api.types.is_bool_dtype(decile.dtype) + and not decile.isna().any(), + f"{IMD_DECILE_COLUMN} must be a complete integer column", + ) + deciles = decile.to_numpy(dtype=np.int64) + _require( + bool( + np.all((deciles >= UNOBSERVED_IMD_DECILE) & (deciles <= len(IMD_DECILES))) + ), + f"{IMD_DECILE_COLUMN} holds codes outside 0 to 10", + ) + observed = household[COUNCIL_TAX_BAND_OBSERVED_COLUMN] + _require( + pd.api.types.is_bool_dtype(observed.dtype) and not observed.isna().any(), + f"{COUNCIL_TAX_BAND_OBSERVED_COLUMN} must be a complete boolean column", + ) + flags = observed.to_numpy(dtype=bool) + bands = household[COUNCIL_TAX_BAND_COLUMN].to_numpy(dtype=object) + _require( + all(band in UK_COUNCIL_TAX_BANDS for band in bands[flags]), + f"an observed {COUNCIL_TAX_BAND_COLUMN} is not a band A to I", + ) + copies = uk_support_channel_copies(household) + decile_inputs = [ + pd.NA if copy or value == UNOBSERVED_IMD_DECILE else str(int(value)) + for value, copy in zip(deciles.tolist(), copies.tolist(), strict=True) + ] + band_inputs = [ + str(band) if flag and not copy else pd.NA + for band, flag, copy in zip( + bands.tolist(), flags.tolist(), copies.tolist(), strict=True + ) + ] + string = pd.StringDtype(storage="python") + return pd.DataFrame( + { + IDENTITY_COLUMN: pd.Series(keys, index=household.index, dtype=string), + REGION_COLUMN: pd.Series(regions, index=household.index, dtype=string), + DECILE_INPUT_COLUMN: pd.Series( + decile_inputs, index=household.index, dtype=string + ), + BAND_INPUT_COLUMN: pd.Series( + band_inputs, index=household.index, dtype=string + ), + }, + index=household.index, + ) + + +def uk_draw_systems(inputs: pd.DataFrame, definition: Mapping) -> np.ndarray: + """Each row's area system, by the definition's region selectors. + + Refuses a row that no system, or more than one, selects, as the shared + operator does. + """ + + spec = validate_assignment_spec(definition) + systems = np.full(len(inputs), "", dtype=object) + claimed = np.zeros(len(inputs), dtype=np.int64) + for system in spec["systems"]: + mask = np.ones(len(inputs), dtype=bool) + for column, accepted in system["selector"].items(): + _require(column in inputs.columns, f"the draw inputs lack {column!r}") + mask &= inputs[column].isin(accepted).to_numpy(dtype=bool) + systems[mask] = system["id"] + claimed += mask + _require( + bool(np.all(claimed == 1)), + "every household must fall in exactly one area system by its region", + ) + return systems + + +def _system_rules(system: Mapping) -> tuple[dict | None, str]: + """A system's ``weight_by`` rule without its default, and the default.""" + + stages = system["stages"] + _require( + len(stages) == 1 and stages[0]["level"] == "area", + "the conditioned draw samples one stage, at the atomic area", + ) + (stage,) = stages + rule = stage.get("weight_by") + if rule is None: + return None, stage["weight"] + return {"input": rule["input"], "map": dict(rule["map"])}, rule["default"] + + +def uk_draw_levels( + inputs: pd.DataFrame, + definition: Mapping, + supports: Mapping[str, AtomicSupport], +) -> np.ndarray: + """Each household's fallback level, by the shared chooser, in row order. + + ``inputs`` is :func:`uk_draw_inputs`' frame and ``definition`` + :func:`uk_conditioned_assignment_definition`'s. Each system's households + go to the chooser with its support and the definition's own arguments: + its constraints as the input-to-support mapping, its stage's + ``weight_by`` rule without the default, and the default as + ``default_weight``; a system without ``weight_by`` (Northern Ireland) + takes its unweighted levels only. So a household's level is the first of + :data:`UK_ATOMIC_FALLBACK_LEVELS` whose inputs are present and whose cell + has mass in the weight it samples by there, and a household with neither + input, an SPI-channel row among them, takes L3. Returns level names. + """ + + spec = validate_assignment_spec(definition) + _require( + isinstance(supports, Mapping) + and set(supports) == {system["id"] for system in spec["systems"]} + and all(isinstance(support, AtomicSupport) for support in supports.values()), + "a decoded support for every system of the definition is required", + ) + systems = uk_draw_systems(inputs, spec) + names = np.full(len(inputs), "", dtype=object) + for system in spec["systems"]: + rows = np.flatnonzero(systems == system["id"]) + if not len(rows): + continue + weight_by, default = _system_rules(system) + levels = ( + UK_ATOMIC_FALLBACK_LEVELS + if weight_by is not None + else tuple( + level for level in UK_ATOMIC_FALLBACK_LEVELS if not level.weight_by + ) + ) + names[rows] = choose_fallback_levels( + supports[system["id"]], + inputs.iloc[rows], + constraint_columns={ + c["input"]: c["support"] for c in system["constraints"] + }, + weight_by=weight_by, + default_weight=default, + levels=levels, + ) + return np.asarray(names.tolist(), dtype=str) + + +def _level_names(inputs: pd.DataFrame, levels: Sequence[str]) -> list[str]: + names = [str(name) for name in np.asarray(levels, dtype=object).tolist()] + _require(len(names) == len(inputs), "one level per household") + _require(set(names) <= set(_LEVELS), "a level outside the UK fallback ladder") + return names + + +def uk_draw_frame(inputs: pd.DataFrame, levels: Sequence[str]) -> pd.DataFrame: + """The draw's inputs at each household's level. + + A copy of ``inputs`` with every input the household's level does not + keep set to missing: the decile where the level keeps the region alone, + the band where the level samples by households. The definition skips a + missing decile and samples a missing band by households, so the draw of + this frame draws each household at its level. + """ + + names = _level_names(inputs, levels) + frame = inputs.copy() + constraints = dict.fromkeys( + column for level in UK_ATOMIC_FALLBACK_LEVELS for column in level.constraints + ) + kept = { + column: np.asarray( + [column in _LEVELS[name].constraints for name in names], dtype=bool + ) + for column in constraints + } + kept[BAND_INPUT_COLUMN] = np.asarray( + [_LEVELS[name].weight_by for name in names], dtype=bool + ) + for column, keep in kept.items(): + if not keep.all(): + frame[column] = frame[column].mask(~keep) + return frame + + +def uk_fallback_reasons( + inputs: pd.DataFrame, levels: Sequence[str], definition: Mapping +) -> pd.DataFrame: + """Why each household is drawn at its level: one status per input. + + Column ``decile``: ``kept`` where the level keeps the decile, + ``unobserved`` where the input is missing, ``dropped`` where it is + present but no level that keeps it has mass. Column ``band`` likewise for + the band's weight, or ``none`` where the household's system weighs no + band (Northern Ireland). An L0 household reads ``kept`` twice. An + SPI-channel row's inputs are missing by rule, so it reads ``unobserved``. + On ``inputs``' index, in its row order. + """ + + names = _level_names(inputs, levels) + spec = validate_assignment_spec(definition) + systems = uk_draw_systems(inputs, spec) + banded = { + system["id"]: _system_rules(system)[0] is not None for system in spec["systems"] + } + decile_present = inputs[DECILE_INPUT_COLUMN].notna().to_numpy(dtype=bool) + band_present = inputs[BAND_INPUT_COLUMN].notna().to_numpy(dtype=bool) + deciles, bands = [], [] + for name, system, has_decile, has_band in zip( + names, + systems.tolist(), + decile_present.tolist(), + band_present.tolist(), + strict=True, + ): + level = _LEVELS[name] + if DECILE_INPUT_COLUMN in level.constraints: + deciles.append("kept") + else: + deciles.append("dropped" if has_decile else "unobserved") + if not banded[system]: + bands.append("none") + elif level.weight_by: + bands.append("kept") + else: + bands.append("dropped" if has_band else "unobserved") + return pd.DataFrame( + { + "decile": pd.Series(deciles, index=inputs.index, dtype=object), + "band": pd.Series(bands, index=inputs.index, dtype=object), + }, + index=inputs.index, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py index 01543433a4..eff34a6b3a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py @@ -10,11 +10,13 @@ every 2022 output area of Scotland in one BRMA, by population-weighted centroid in the published BRMA polygons (Scotland corrected by Rent Service Scotland's postcode lookup); ``tools/build_uk_brma_area_layer.py`` rebuilds -it from Chronicle's raw registrations. The post-geography node -``uk.full.geography.brma`` reads a placed household's BRMA off its assigned -area, as its local authority and constituency are read. Northern Ireland has -no open BRMA geography and keeps the spine's draw. The national line has no -areas and keeps the spine's draw everywhere. +it from Chronicle's raw registrations. The clone-first spine's BRMA stage +(``spine_geography.UKAreaBRMAStageTransform``, G4 of microcosm#931) reads each +Great Britain household's BRMA off the area the spine draws it, as its local +authority and constituency are read. Northern Ireland has no open BRMA +geography: its households keep the BRMA ``frs_brma`` draws from the census +counts within their region, keyed on each benefit unit. A spine without areas +keeps that regional draw everywhere. Everything here is pure and depends on the standard library, numpy and pandas only, so the build tool can load this module by path. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_brma.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_brma.py index 262cb0ed16..862bb5a15d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_brma.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_brma.py @@ -1,4 +1,17 @@ -"""UK FRS BRMA stochastic assignment from committed census household counts.""" +"""UK FRS BRMA stochastic assignment from committed census household counts. + +Each benefit unit draws a BRMA from the census count-table cell of its region +and ``LHA_category``, with a uniform keyed on its own id, and each household +picks one of its benefit units' BRMAs with a uniform keyed on its own id +(:func:`draw_household_brmas`). Every uniform is keyed on its own id alone, +so drawing a subset of a frame's households gives each the BRMA the whole +frame's draw gives it. ``spine_geography``'s BRMA stage draws Northern Ireland's +households this way, and its SPI-copy area stage the Northern Ireland copies, +each on its own benefit units (microcosm#931). + +Refusals name columns and declared cells only, never a household's or a +benefit unit's id. +""" from __future__ import annotations @@ -28,6 +41,13 @@ FRS_BRMA_OUTPUT_COLUMNS = ("brma",) UK_BRMA_PREDICTORS = ("LHA_category",) UK_BRMA_DECLARED_SEEDS = {"brma": 0} +#: The committed census counts the draw samples from. +UK_BRMA_COUNT_RESOURCE = "brma_private_rented_households.json" +#: The salts each uniform is keyed on with the seed and an id: each benefit +#: unit's draw from its cell, and each household's pick among its benefit +#: units' BRMAs. +UK_BRMA_BENUNIT_SALT = "brma" +UK_BRMA_HOUSEHOLD_PICK_SALT = "brma:household_pick" @dataclass(frozen=True) @@ -59,9 +79,7 @@ def output_columns() -> tuple[str, ...]: def load_brma_count_resource() -> Mapping[str, Any]: return json.loads( - files("microcosm.build.uk") - .joinpath("brma_private_rented_households.json") - .read_text() + files("microcosm.build.uk").joinpath(UK_BRMA_COUNT_RESOURCE).read_text() ) @@ -74,22 +92,14 @@ def add_frs_brma( person = frame.table("person").copy() benunit = frame.table("benunit").copy() household = frame.table("household").copy() - if len(lha_category) != len(benunit): - raise ValueError("LHA_category materialization must align to benunit rows.") - benunit["LHA_category"] = [_enum_name(value) for value in lha_category] - benunit["region"] = _benunit_regions(person, household, benunit) - benunit["brma"] = assign_brma_by_cell( - benunit, - count_resource=count_resource, - seed=UK_BRMA_DECLARED_SEEDS["brma"], - ) - household["brma"] = collapse_benunit_brma_to_household( + household["brma"] = draw_household_brmas( person, - benunit[["benunit_id", "brma"]], + benunit, household, + lha_category=lha_category, + count_resource=count_resource, seed=UK_BRMA_DECLARED_SEEDS["brma"], ) - benunit = benunit.drop(columns=["LHA_category", "region", "brma"]) result = uk_national_frame( person=person, benunit=benunit, @@ -103,6 +113,50 @@ def add_frs_brma( return result +def draw_household_brmas( + person: pd.DataFrame, + benunit: pd.DataFrame, + household: pd.DataFrame, + *, + lha_category: Sequence[object], + count_resource: Mapping[str, Any], + seed: int = UK_BRMA_DECLARED_SEEDS["brma"], +) -> np.ndarray: + """Each of ``household``'s rows' BRMA, drawn on its own benefit units. + + ``lha_category`` is aligned to ``benunit``'s rows, and ``person`` places + each benefit unit in its household. ``household`` may hold any subset of + the frame's households: the draw reads only the benefit units of its + rows. Each of them samples a BRMA from its region and ``LHA_category`` + cell with the uniform keyed on its own id (:data:`UK_BRMA_BENUNIT_SALT`), + and each household picks one of its benefit units' BRMAs, in id order, + with the uniform keyed on its own id (:data:`UK_BRMA_HOUSEHOLD_PICK_SALT`), + both under ``seed``. So a subset's households take the BRMAs a draw of the + whole frame gives them, and a copy keyed on ids of its own takes a draw of + its own. Returns an object array aligned to ``household``'s rows. + + Refused, naming no id: a ``lha_category`` not aligned to ``benunit``, a + benefit unit split across households, a household with no benefit unit, + and a count-table cell the resource lacks (by region and category). + """ + + if len(lha_category) != len(benunit): + raise ValueError("LHA_category materialization must align to benunit rows.") + owners = benunit["benunit_id"].map(_benunit_household_map(person)) + drawn = owners.isin(household["household_id"]).to_numpy(dtype=bool) + units = benunit.loc[drawn, ["benunit_id"]].copy() + units["LHA_category"] = [ + _enum_name(value) + for value, keep in zip(lha_category, drawn.tolist(), strict=True) + if keep + ] + units["region"] = _benunit_regions(person, household, units) + units["brma"] = assign_brma_by_cell(units, count_resource=count_resource, seed=seed) + return collapse_benunit_brma_to_household( + person, units[["benunit_id", "brma"]], household, seed=seed + ) + + def assign_brma_by_cell( benunit: pd.DataFrame, *, @@ -111,7 +165,7 @@ def assign_brma_by_cell( ) -> np.ndarray: cells = count_resource["cells"] draws = stable_identity_uniforms( - benunit["benunit_id"].to_numpy(), seed=seed, salt="brma" + benunit["benunit_id"].to_numpy(), seed=seed, salt=UK_BRMA_BENUNIT_SALT ) assigned: list[str] = [] for index, row in enumerate(benunit.itertuples(index=False)): @@ -146,7 +200,7 @@ def collapse_benunit_brma_to_household( values["household_id"] = values["benunit_id"].map(mapping) household_ids = household["household_id"].to_numpy() draws = stable_identity_uniforms( - household_ids, seed=seed, salt="brma:household_pick" + household_ids, seed=seed, salt=UK_BRMA_HOUSEHOLD_PICK_SALT ) options_by_household = { household_id: group.sort_values("benunit_id")["brma"].astype(str).to_list() @@ -156,7 +210,10 @@ def collapse_benunit_brma_to_household( for index, household_id in enumerate(household_ids): options = options_by_household.get(household_id) if not options: - raise ValueError(f"household {household_id!r} has no benunit BRMA options.") + raise ValueError( + "Each household picks its BRMA among its benefit units' draws; " + "a household has no benefit unit with a drawn BRMA." + ) pick = min(int(draws[index] * len(options)), len(options) - 1) assigned.append(options[pick]) return np.asarray(assigned, dtype=object) @@ -179,12 +236,9 @@ def _benunit_regions( def _benunit_household_map(person: pd.DataFrame) -> pd.Series: placements = person[["person_benunit_id", "person_household_id"]].drop_duplicates() if placements["person_benunit_id"].duplicated().any(): - duplicated = placements.loc[ - placements["person_benunit_id"].duplicated(keep=False), - "person_benunit_id", - ].unique() raise ValueError( - f"benunit id(s) split across households: {duplicated[:5].tolist()}." + "A benefit unit's members must share one household; the person table " + "splits a benefit unit across households." ) return placements.set_index("person_benunit_id")["person_household_id"] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py index 9ec0a511c4..3068b51d60 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py @@ -11,11 +11,14 @@ excluded units, beside the notice's own districts (``tools/build_uk_rfdr_area_layer.py``). The tool reads the ONS postcode lookup and flags every 2021 output area of England and Wales and 2022 output area of Scotland whose postcodes mostly lie in the scheme; -``uk/rfdr_area_layer.json`` lists them. The post-geography node -``uk.full.geography.rfdr`` sets the flag on each placed household from its -assigned output area. Northern Ireland has no area in the scheme. The -national line has no areas, so it leaves the engine's default of false -(María's ruling of 2026-10-28). +``uk/rfdr_area_layer.json`` lists them. The clone-first spine's fuel-relief +stage (``spine_geography.UKRFDRStageTransform``, G5 of microcosm#931) sets the +flag on each household from the output area the spine draws it, and holds +the layer to the spine's supports: every output area the layer flags must be +an area of its system's support (:func:`area_flags`' ``roster``). Northern +Ireland has no area in the scheme. A dataset built from a spine without +areas carries no flag and keeps the engine's default of false, which María's +ruling of 2026-10-28 set for the national line. Everything here is pure and depends on the standard library, numpy and pandas only, so the build tool can load this module by path. @@ -245,29 +248,59 @@ def area_flags( area_system: Iterable[object], area_code: Iterable[object], require_flagged: bool = True, + roster: Mapping[str, Iterable[object]] | None = None, ) -> tuple[np.ndarray, dict[str, int]]: """Each household's flag from its assigned output area. A Great Britain household is in the scheme when the layer lists its area; a Northern Ireland household never is. Any other support system refuses, since the layer cannot say where it is. A Great Britain code that is not a - well-formed output-area code of its system refuses too, and so does a - system (England and Wales, or Scotland) with households none of which is - flagged: either would let a change of code format or vintage publish its - households unflagged (Anthony's reviews of microcosm#1420). The zero-flag - refusal holds only with ``require_flagged``, which a full-scale build - sets: a sampled rung places so few England and Wales households (41 of - 188,880 output areas are in the scheme) that it can miss them all by - chance (Vahid's review and the independent review of microcosm#1420). + well-formed output-area code of its system refuses too: it would let a + change of code format publish its households unflagged (Anthony's reviews + of microcosm#1420). Against a change of code vintage, a caller holds the + layer to either check, or both: + + - ``roster`` gives the areas of each Great Britain system's support. + Every output area the layer flags must be one of its system's, so a + layer and a support of different vintages refuse whatever the + households drew. The clone-first spine's fuel-relief stage passes its + supports' areas (microcosm#931, C-6). + - ``require_flagged`` refuses a system (England and Wales, or Scotland) + with households none of which is flagged. It suits a full-scale build + only: a sampled rung places so few England and Wales households (41 of + 188,880 output areas are in the scheme) that it can miss them all by + chance (Vahid's review and the independent review of microcosm#1420). + + The unknown-system, roster and malformed-code refusals name no value of + a household and no code, and count nothing. Returns the flags and exact + row counts by system; callers round counts before any receipt. """ systems = np.asarray(list(area_system), dtype=object) codes = np.asarray(list(area_code), dtype=object) if len(systems) != len(codes): raise ValueError("area_system and area_code must align.") - unknown = sorted({str(system) for system in systems} - {*SYSTEMS, NI_SYSTEM}) - if unknown: - raise ValueError(f"RFDR flags read no support system {unknown}.") + known = {*SYSTEMS, NI_SYSTEM} + if any(str(system) not in known for system in systems): + raise ValueError( + f"RFDR flags read households of the support systems {sorted(known)} " + "only; area_system holds another value." + ) + if roster is not None: + if not isinstance(roster, Mapping) or any( + system not in roster for system in SYSTEMS + ): + raise ValueError( + "RFDR flags hold the layer to a roster of support areas for each " + f"Great Britain system {sorted(SYSTEMS)}." + ) + for system, flagged in layer.areas.items(): + if not flagged <= {str(code) for code in roster[system]}: + raise ValueError( + f"The RFDR layer flags {system} output areas that system's " + "support does not hold: the layer and the support must carry " + "one code vintage (microcosm#931)." + ) flags = np.zeros(len(codes), dtype=bool) counts: dict[str, int] = {} for system, flagged in layer.areas.items(): @@ -280,8 +313,8 @@ def area_flags( ] if malformed: raise ValueError( - f"RFDR flags read {len(malformed)} {system} code(s) that are not " - f"well-formed output-area codes (first: {malformed[0]!r})." + f"RFDR flags read {system} code(s) that are not well-formed " + "output-area codes." ) flags[rows] = np.asarray([code in flagged for code in codes[rows]], dtype=bool) if require_flagged and rows.any() and not flags[rows].any(): diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_geography.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_geography.py new file mode 100644 index 0000000000..33ded457f6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_geography.py @@ -0,0 +1,2043 @@ +"""The clone-first spine's area stages (microcosm#931). + +Once the geography identity has keyed every geographic clone (G1b), these +stages give each clone a place of its own, so every later stage reads a +clone's area as its household's: + +- **G2, ``atomic_area_assign``** (:class:`UKAtomicAreaAssignStageTransform`) + draws each household's atomic area within its FRS region with the shared + operator, conditioned on its IMD decile and observed council-tax band + (:mod:`~microcosm.build.uk_runtime.atomic_area_conditioning`): the chooser + picks each household's fallback level, then ``assign_atomic``, + ``derive_geography`` and ``validate_geography`` run on one narrow, + transient frame of draw inputs, which carries no area column of its own. + It writes the three assignment columns and the ten ladder columns in the + graph's string dtype (``StringDtype("python")``), and refuses when the + validation fails. +- **G3, ``local_authority``** (:class:`UKLocalAuthorityStageTransform`) + resolves the engine's ``local_authority`` from the drawn + ``local_authority_code`` through the committed names roster, fail-closed, + never defaulted (microcosm#953), and writes it in the graph's string dtype, + as the dense line's node did, not as the resolver's object column. +- **G4, ``area_brma``** (:class:`UKAreaBRMAStageTransform`), in + ``frs_brma``'s slot after ``frs_household_draws``, writes each + household's ``brma``. + Great Britain households read it off their drawn LSOA (England and Wales) + or output area (Scotland) through the committed area-to-BRMA layer, + fail-closed on an area the layer does not hold or places in another FRS + region. Northern Ireland has no open BRMA geography: one engine call over + the whole frame materialises ``LHA_category``, and its benefit units draw + from the census count table with ``frs_brma``'s keys, seed and salts + (:func:`~microcosm.build.uk_runtime.frs_brma.draw_household_brmas`), so + each takes the BRMA ``frs_brma`` gives it. ``brma`` stays an object + column, as ``frs_brma`` writes it. +- **G5, ``rfdr``** (:class:`UKRFDRStageTransform`) sets each household's + ``in_rural_fuel_duty_relief_area`` from its drawn output area through the + committed RFDR layer. It refuses a layer that flags an output area outside + its system's support, so the layer and the supports must share one code + vintage; a system with households none of which is flagged is reported, as + a boolean, not refused (C-6). +- **The SPI-copy area stage, ``spi_copy_areas``** + (:class:`UKSPICopyAreasStageTransform`), after the SPI support channel and + its income band donors (``spi_support_channel`` and + ``spi_income_band_donors``), gives every row those stages copied a key and + a place of its own, where the copy had its parent's: a geography key over + its own copy path, an area drawn at L3 (its region, by households) with + G2's definition, then G3's local authority, G4's BRMA and G5's flag. Its + Northern Ireland copies draw their BRMA from the census counts on their + own benefit units, with G4's seed and salts, after one engine call over + their households alone materialises their ``LHA_category`` (C-9, María's + ruling of 2026-11-11). That call is one simulation over the copies of + every geographic clone together, as it is at one clone, never one per + clone: the ruling rests on ``LHA_category`` reading only a household's own + bedroom entitlement and shared-rate status. Every other row is left as it + was. + +G2, G3 and G5 compute with keyed integer draws, exact integer sampling +masses and string lookups, so each is bitwise, the class of every spine stage +(C-2). G4 and the copy stage read ``LHA_category`` from the rules engine and +draw Northern Ireland's BRMAs as ``frs_brma`` does. No stage declares them +yet: each transform checks the stage it runs as against the reviewed +declaration this module states (name, household grain, outputs, rewrites, +the reviewed operations verbatim and its artifacts), which the declarations +will carry. + +This module's refusals give columns, dtypes and declared constants only, +never a key, an id, a code of a household, a value or a count of households. +The evidence counts distinct true source households through +``receipt_counts`` (a household's clones and copies counted once), never per +area or per cell, and prints no key or code of a household (C-44): G2's +sources by system, by level and by fallback reason, G3's sources, and G4's +sources by grain and Northern Ireland's sources drawn, over every clone, +since each is the same at every clone count; G5's flagged sources and the +copy stage's copied, donor and flagged sources among clone-0 rows, where a +union could grow with K. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from importlib.resources import files +from types import MappingProxyType +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.atomic_geography import ( + AtomicSupport, + assign_atomic, + decode_atomic_support, + derive_geography, + validate_assignment_spec, + validate_geography, +) +from microcosm.build.source_lineage import copy_paths +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.atomic_area_conditioning import ( + BAND_INPUT_COLUMN, + COUNCIL_TAX_BAND_COLUMN, + COUNCIL_TAX_BAND_OBSERVED_COLUMN, + DECILE_INPUT_COLUMN, + DEFAULT_WEIGHT_COLUMN, + IDENTITY_COLUMN, + REGION_COLUMN, + SUPPORT_CHANNEL_BRANCH, + UK_ATOMIC_FALLBACK_LEVELS, + UK_DRAW_CONSTRAINTS, + UK_FALLBACK_STATUSES, + UK_SPINE_AREA_COLUMNS, + UK_SPINE_ATOMIC_AREA_SEED, + UK_SPINE_ATOMIC_AREA_STREAM, + UK_SPINE_LADDER_COLUMNS, + UNOBSERVED_IMD_DECILE, + uk_band_weight_map, + uk_conditioned_assignment_definition, + uk_conditioned_supports, + uk_draw_frame, + uk_draw_inputs, + uk_draw_levels, + uk_fallback_reasons, +) +from microcosm.build.uk_runtime.atomic_area_support import ( + ASSIGNMENT_COLUMNS, + CONDITIONED_SOURCES, + IMD_DECILE_COLUMN, + SYSTEMS, +) +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.brma_area_layer import ( + GRAINS as BRMA_GRAINS, +) +from microcosm.build.uk_runtime.brma_area_layer import ( + METHOD_VERSION as BRMA_METHOD_VERSION, +) +from microcosm.build.uk_runtime.brma_area_layer import ( + NI_SYSTEM, + BRMAAreaLayer, + area_brmas, + load_brma_area_layer, +) +from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations +from microcosm.build.uk_runtime.frs_brma import ( + FRS_BRMA_OUTPUT_COLUMNS, + UK_BRMA_BENUNIT_SALT, + UK_BRMA_COUNT_RESOURCE, + UK_BRMA_DECLARED_SEEDS, + UK_BRMA_HOUSEHOLD_PICK_SALT, + UK_BRMA_PREDICTORS, + draw_household_brmas, +) +from microcosm.build.uk_runtime.geographic_clones import ( + SOURCE_YEAR_COLUMN, + geographic_clone_count, + ladder_clone_index_column, +) +from microcosm.build.uk_runtime.geographic_identity import ( + FRS_RELEASE_ARTIFACT_ROLE, + FRS_RELEASE_RESOURCE, + GEOGRAPHIC_IDENTITY_SOURCE, + frs_vintages_by_source_year, +) +from microcosm.build.uk_runtime.geography_sources import LAD23_NAMES_SHA256 +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_SUPPORT_BRANCH, + UK_COPY_LINEAGE, +) +from microcosm.build.uk_runtime.local_authority_input import ( + UK_LOCAL_AUTHORITY_CODE_COLUMN, + UK_LOCAL_AUTHORITY_INPUT_COLUMN, + UK_LOCAL_AUTHORITY_NAMES_RESOURCE, + UK_LOCAL_AUTHORITY_VINTAGE, + local_authority_engine_key_by_code, + resolve_local_authority_engine_keys, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.receipt_counts import ( + distinct_carriers, + record_source_households, + rounded_count, +) +from microcosm.build.uk_runtime.rfdr_area_layer import ( + METHOD_VERSION as RFDR_METHOD_VERSION, +) +from microcosm.build.uk_runtime.rfdr_area_layer import ( + RFDR_COLUMN, + RFDRAreaLayer, + area_flags, + load_rfdr_area_layer, +) +from microcosm.build.uk_runtime.rfdr_area_layer import ( + SYSTEMS as RFDR_SYSTEMS, +) +from microcosm.frame import Frame, MassChangeRecord, Weights +from microcosm.frame.rules import assert_rules_engine_country +from microcosm.graph.canonical import canonical_json + +__all__ = [ + "AREA_BRMA_OPERATION_KIND", + "AREA_BRMA_STAGE_NAME", + "ATOMIC_AREA_ASSIGN_OPERATION_KIND", + "ATOMIC_AREA_ASSIGN_STAGE_NAME", + "BRMA_COUNT_ARTIFACT_ROLE", + "BRMA_LAYER_ARTIFACT_ROLE", + "BRMA_LAYER_RESOURCE", + "LOCAL_AUTHORITY_ARTIFACT_ROLE", + "LOCAL_AUTHORITY_OPERATION_KIND", + "LOCAL_AUTHORITY_STAGE_NAME", + "RFDR_ARTIFACT_ROLE", + "RFDR_LAYER_RESOURCE", + "RFDR_OPERATION_KIND", + "RFDR_STAGE_NAME", + "SPI_COPY_AREAS_OPERATION_KIND", + "SPI_COPY_AREAS_STAGE_NAME", + "UK_AREA_BRMA_OUTPUT_COLUMNS", + "UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS", + "UK_LOCAL_AUTHORITY_OUTPUT_COLUMNS", + "UK_RFDR_OUTPUT_COLUMNS", + "UK_SPI_COPY_AREAS_REWRITE_COLUMNS", + "UKAreaBRMAResult", + "UKAreaBRMAStageTransform", + "UKAtomicAreaAssignResult", + "UKAtomicAreaAssignStageTransform", + "UKLocalAuthorityResult", + "UKLocalAuthorityStageTransform", + "UKRFDRResult", + "UKRFDRStageTransform", + "UKSPICopyAreasResult", + "UKSPICopyAreasStageTransform", + "UKSpineAreaDraw", + "area_brma_operation_parameters", + "assign_uk_spi_copy_areas", + "assign_uk_spine_areas", + "assign_uk_spine_brmas", + "atomic_area_assign_operation_parameters", + "draw_uk_spine_areas", + "flag_uk_spine_rfdr", + "load_committed_brma_counts", + "load_committed_brma_layer", + "load_committed_rfdr_layer", + "local_authority_operation_parameters", + "resolve_uk_spine_local_authorities", + "rfdr_operation_parameters", + "spi_copy_areas_operation_parameters", + "uk_area_brmas", + "uk_local_authorities", + "uk_rfdr_flags", + "uk_spi_copy_keys", +] + +_STRING = pd.StringDtype(storage="python") +_CLONE_INDEX_COLUMN = ladder_clone_index_column("household") + + +# --- Shared helpers --------------------------------------------------------------- + + +def _assert_stage( + stage: SourceStageSpec, + *, + name: str, + what: str, + outputs: tuple[str, ...], + operations: tuple[tuple[str, dict[str, object]], ...], + artifacts: tuple[tuple[str, str], ...] = (), + rewrites: tuple[str, ...] = (), +) -> None: + """The stage a transform runs as: ``name``, household grain, exactly + ``outputs`` and ``rewrites`` (none unless given), the reviewed + ``operations`` verbatim, and exactly ``artifacts`` (role, resource), each + ``runtime_sha256_required``.""" + + if not isinstance(stage, SourceStageSpec): + raise TypeError(f"{what} runs as a SourceStageSpec stage.") + if stage.stage != name: + raise ValueError(f"Expected stage {name!r}, got {stage.stage!r}.") + if stage.grain != "household": + raise ValueError( + f"{what} writes households; grain must be 'household', got {stage.grain!r}." + ) + if tuple(stage.outputs) != outputs or tuple(stage.rewrites) != rewrites: + raise ValueError( + f"{what} writes exactly its own columns; the stage must declare " + f"outputs {outputs} and rewrites {rewrites}, got " + f"{tuple(stage.outputs)} / {tuple(stage.rewrites)}." + ) + _assert_closed_world_operations(stage, operations) + declared = sorted( + (str(artifact.get("role")), str(artifact.get("resource"))) + for artifact in stage.artifacts + if artifact.get("runtime_sha256_required") is True + ) + if len(stage.artifacts) != len(artifacts) or declared != sorted(artifacts): + raise ValueError( + f"{what} reads the artifacts {list(artifacts)} (role, resource), each " + "with runtime_sha256_required: true, and no other; the stage " + f"declares {len(stage.artifacts)} artifact(s)." + ) + + +def _require_columns( + household: pd.DataFrame, columns: Sequence[str], what: str +) -> None: + missing = [column for column in columns if column not in household.columns] + if missing: + raise ValueError( + f"{what} reads household {missing}, which earlier stages write; " + "the frame lacks them (microcosm#931)." + ) + + +def _refuse_written(household: pd.DataFrame, columns: Sequence[str], what: str) -> None: + present = [column for column in columns if column in household.columns] + if present: + raise ValueError( + f"The frame already carries {present}; {what} writes each household's " + "once (microcosm#931)." + ) + + +def _with_household(frame: Frame, household: pd.DataFrame) -> Frame: + """``frame`` with ``household`` in place of its household table; ids, + weights, strata, the mass log and the metadata are the frame's own.""" + + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = household + # The household weights pass through unchanged. Annotated so the stage's + # static walk meets microcosm.frame's Weights module. + weights: Weights = frame.weights_for("household") + result = Frame( + tables, + frame.schema, + {"household": weights}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + validate_uk_national_frame(result) + return result + + +def _clone_zero(household: pd.DataFrame) -> np.ndarray: + """Rows of geographic clone 0: every row of a table without the index.""" + + if _CLONE_INDEX_COLUMN not in household.columns: + return np.ones(len(household), dtype=bool) + return household[_CLONE_INDEX_COLUMN].to_numpy(dtype=np.int64) == 0 + + +def _sources_by( + sources: np.ndarray, labels: np.ndarray, names: Sequence[str] +) -> dict[str, int]: + """Distinct true sources among the rows carrying each label, exact.""" + + return {name: distinct_carriers(sources[labels == name].tolist()) for name in names} + + +# --- G2, atomic_area_assign ------------------------------------------------------- + +ATOMIC_AREA_ASSIGN_STAGE_NAME = "atomic_area_assign" +ATOMIC_AREA_ASSIGN_OPERATION_KIND = "assign_atomic_area" +#: The columns G2 writes: the three assignment columns, then the ten ladder +#: columns, every one in the graph's string dtype. +UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS = UK_SPINE_AREA_COLUMNS +_VALIDATION_SCOPE = "atomic_geography_mapping_integrity" + + +def atomic_area_assign_operation_parameters() -> tuple[ + tuple[str, dict[str, object]], ... +]: + """The reviewed operation G2's stage must declare verbatim: the draw's + key, sources, constraints, per-household weight, inputs, fallback ladder, + layers, stream and seed, as + :func:`~microcosm.build.uk_runtime.atomic_area_conditioning.uk_conditioned_assignment_definition` + and :func:`~microcosm.build.uk_runtime.atomic_area_conditioning.uk_draw_inputs` + use them.""" + + return ( + ( + ATOMIC_AREA_ASSIGN_OPERATION_KIND, + { + "identity": IDENTITY_COLUMN, + "sources": {system: CONDITIONED_SOURCES[system] for system in SYSTEMS}, + "constraints": [ + { + "input": column, + "support": matched, + "required": column == REGION_COLUMN, + } + for column, matched in UK_DRAW_CONSTRAINTS.items() + ], + "weight_by": { + "input": BAND_INPUT_COLUMN, + "default": DEFAULT_WEIGHT_COLUMN, + "bands": { + system: sorted(uk_band_weight_map(system)) for system in SYSTEMS + }, + }, + "draw_inputs": { + DECILE_INPUT_COLUMN: { + "column": IMD_DECILE_COLUMN, + "unobserved": UNOBSERVED_IMD_DECILE, + }, + BAND_INPUT_COLUMN: { + "column": COUNCIL_TAX_BAND_COLUMN, + "observed": COUNCIL_TAX_BAND_OBSERVED_COLUMN, + }, + "missing_on_copies_of": SUPPORT_CHANNEL_BRANCH, + }, + "fallback_levels": [ + { + "name": level.name, + "constraints": list(level.constraints), + "weight_by": level.weight_by, + } + for level in UK_ATOMIC_FALLBACK_LEVELS + ], + "layers": list(UK_SPINE_LADDER_COLUMNS), + "outputs": list(ASSIGNMENT_COLUMNS), + "stream": UK_SPINE_ATOMIC_AREA_STREAM, + "seed": UK_SPINE_ATOMIC_AREA_SEED, + }, + ), + ) + + +def _assert_atomic_area_assign_stage(stage: SourceStageSpec) -> None: + _assert_stage( + stage, + name=ATOMIC_AREA_ASSIGN_STAGE_NAME, + what="The area draw", + outputs=UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS, + operations=atomic_area_assign_operation_parameters(), + ) + + +@dataclass(frozen=True) +class UKSpineAreaDraw: + """The draw on one household table: G2's columns, and each row's level + and fallback reason. Nothing here prints in the repr but the digests.""" + + columns: pd.DataFrame = field(repr=False) + levels: np.ndarray = field(repr=False) + reasons: pd.DataFrame = field(repr=False) + definition_sha256: str + support_sha256: Mapping[str, str] + + +def draw_uk_spine_areas( + household: pd.DataFrame, + *, + definition: Mapping[str, object], + supports: Mapping[str, AtomicSupport], +) -> UKSpineAreaDraw: + """Every household row's area, ladder, level and fallback reason. + + Builds the narrow draw frame (:func:`uk_draw_inputs`), picks each row's + level (:func:`uk_draw_levels`), sets the inputs its level drops to missing + (:func:`uk_draw_frame`), then runs ``assign_atomic``, ``derive_geography`` + and ``validate_geography`` on that one frame, which carries no area column + until the draw adds its own. Returns the three assignment columns and the + ten ladder columns in the graph's string dtype, on ``household``'s index + and in its row order. Each row's draw is keyed on its geography key alone, + so a row's area does not depend on the table's other rows or their order. + + Refused: what the adapter and the chooser refuse; a null or repeated key + and a failed validation, by the shared operator's count-free texts. + """ + + inputs = uk_draw_inputs(household) + levels = uk_draw_levels(inputs, definition, supports) + draw = uk_draw_frame(inputs, levels) + assigned = assign_atomic(draw, definition, supports) + placed = pd.concat([draw, assigned], axis=1) + derived = derive_geography(placed, definition, supports) + if set(derived.columns) != set(UK_SPINE_LADDER_COLUMNS): + raise ValueError( + "The area draw derives exactly the ten ladder columns " + f"{list(UK_SPINE_LADDER_COLUMNS)}; the definition derives others." + ) + receipt = validate_geography( + pd.concat([placed, derived], axis=1), definition, supports + ) + if receipt.get("outcome") != "pass" or receipt.get("scope") != _VALIDATION_SCOPE: + raise ValueError("The area draw's mapping-integrity validation did not pass.") + values = pd.concat([assigned, derived], axis=1) + columns = pd.DataFrame( + { + column: pd.Series( + values[column].to_numpy(dtype=object), + index=household.index, + dtype=_STRING, + ) + for column in UK_SPINE_AREA_COLUMNS + }, + index=household.index, + ) + return UKSpineAreaDraw( + columns=columns, + levels=levels, + reasons=uk_fallback_reasons(inputs, levels, definition), + definition_sha256=str(receipt["definition_sha256"]), + support_sha256=MappingProxyType( + dict(sorted(receipt["support_sha256"].items())) + ), + ) + + +@dataclass(frozen=True) +class UKAtomicAreaAssignResult: + """G2's frame and what its evidence reads of it. + + The source counts are exact distinct true source households; the + evidence gives them rounded, and none prints in the repr. + """ + + frame: Frame = field(repr=False) + clone_count: int + seed: int + stream: tuple[object, ...] + definition_sha256: str + support_sha256: Mapping[str, str] + source_households: int = field(repr=False) + by_system: Mapping[str, int] = field(repr=False) + by_level: Mapping[str, int] = field(repr=False) + by_reason: Mapping[str, Mapping[str, int]] = field(repr=False) + + def evidence(self) -> dict[str, object]: + # A result exists only for a draw whose validation passed. + return { + "stage": ATOMIC_AREA_ASSIGN_STAGE_NAME, + "clone_count": int(self.clone_count), + "seed": int(self.seed), + "stream": list(self.stream), + "definition_sha256": self.definition_sha256, + "support_sha256": dict(self.support_sha256), + "validation": {"outcome": "pass", "scope": _VALIDATION_SCOPE}, + "fallback_levels": [level.name for level in UK_ATOMIC_FALLBACK_LEVELS], + "source_households": rounded_count(self.source_households), + "source_households_by_system": { + system: rounded_count(count) for system, count in self.by_system.items() + }, + "source_households_by_level": { + level: rounded_count(count) for level, count in self.by_level.items() + }, + "source_households_by_fallback_reason": { + column: { + status: rounded_count(count) for status, count in statuses.items() + } + for column, statuses in self.by_reason.items() + }, + "outputs": list(UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS), + } + + +def assign_uk_spine_areas( + frame: Frame, + *, + definition: Mapping[str, object], + supports: Mapping[str, AtomicSupport], +) -> UKAtomicAreaAssignResult: + """``frame`` with every household's area and ladder (G2). + + ``definition`` is + :func:`~microcosm.build.uk_runtime.atomic_area_conditioning.uk_conditioned_assignment_definition`'s + and ``supports`` the decoded supports it declares. Rows, ids, weights, + the mass log and the metadata are the frame's own. Refused, by name: a + frame already carrying any column G2 writes, and every refusal of + :func:`draw_uk_spine_areas`. + """ + + if not isinstance(frame, Frame): + raise TypeError("The area draw places the households of a Frame.") + validate_uk_national_frame(frame) + household = frame.table("household") + _refuse_written(household, UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS, "the area draw") + draw = draw_uk_spine_areas(household, definition=definition, supports=supports) + placed = household.copy() + for column in UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS: + placed[column] = draw.columns[column] + result = _with_household(frame, placed) + sources = record_source_households( + result, "household", stage=ATOMIC_AREA_ASSIGN_STAGE_NAME + ) + systems = draw.columns[ASSIGNMENT_COLUMNS[1]].to_numpy(dtype=object) + stream = tuple(definition["stream"]) + return UKAtomicAreaAssignResult( + frame=result, + clone_count=geographic_clone_count(household), + seed=int(stream[3]), + stream=stream, + definition_sha256=draw.definition_sha256, + support_sha256=draw.support_sha256, + source_households=distinct_carriers(sources.tolist()), + by_system=_sources_by(sources, systems, SYSTEMS), + by_level=_sources_by( + sources, draw.levels, [level.name for level in UK_ATOMIC_FALLBACK_LEVELS] + ), + by_reason={ + column: _sources_by( + sources, draw.reasons[column].to_numpy(dtype=object), statuses + ) + for column, statuses in UK_FALLBACK_STATUSES.items() + }, + ) + + +@dataclass(frozen=True) +class UKAtomicAreaAssignStageTransform: + """Whole-stage transform for the area draw (G2). + + ``supports`` holds the three conditioned supports' bytes, keyed by + system. Constructing the transform checks the stage it runs as and builds + the definition, which holds each support to canonical bytes, with the + declared seed; running it calls :func:`assign_uk_spine_areas`. + """ + + stage: SourceStageSpec + supports: Mapping[str, bytes] = field(repr=False) + definition: Mapping[str, object] = field(init=False, repr=False, compare=False) + decoded_supports: Mapping[str, AtomicSupport] = field( + init=False, repr=False, compare=False + ) + last_result: UKAtomicAreaAssignResult | None = field( + default=None, init=False, repr=False + ) + + def __post_init__(self) -> None: + _assert_atomic_area_assign_stage(self.stage) + # The definition holds every support to canonical bytes, so the + # shared decoder alone reads them after it. + definition = uk_conditioned_assignment_definition( + self.supports, seed=UK_SPINE_ATOMIC_AREA_SEED + ) + decoded = { + system: decode_atomic_support(self.supports[system]) for system in SYSTEMS + } + object.__setattr__(self, "definition", definition) + object.__setattr__(self, "decoded_supports", MappingProxyType(decoded)) + + def __call__(self, frame: Frame) -> Frame: + _assert_atomic_area_assign_stage(self.stage) + result = assign_uk_spine_areas( + frame, definition=self.definition, supports=self.decoded_supports + ) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +# --- G3, local_authority ---------------------------------------------------------- + +LOCAL_AUTHORITY_STAGE_NAME = "local_authority" +LOCAL_AUTHORITY_OPERATION_KIND = "resolve_local_authority" +#: The stage's one artifact: the committed ONS names roster, whose bytes enter +#: the stage's contract. +LOCAL_AUTHORITY_ARTIFACT_ROLE = "local_authority_names" +UK_LOCAL_AUTHORITY_OUTPUT_COLUMNS = (UK_LOCAL_AUTHORITY_INPUT_COLUMN,) + + +def local_authority_operation_parameters() -> tuple[tuple[str, dict[str, object]], ...]: + """The reviewed operation G3's stage must declare verbatim: the code it + reads, the engine input it writes, the roster and the roster's vintage.""" + + return ( + ( + LOCAL_AUTHORITY_OPERATION_KIND, + { + "code_column": UK_LOCAL_AUTHORITY_CODE_COLUMN, + "output": UK_LOCAL_AUTHORITY_INPUT_COLUMN, + "names_resource": UK_LOCAL_AUTHORITY_NAMES_RESOURCE, + "vintage": UK_LOCAL_AUTHORITY_VINTAGE, + }, + ), + ) + + +def _assert_local_authority_stage(stage: SourceStageSpec) -> None: + _assert_stage( + stage, + name=LOCAL_AUTHORITY_STAGE_NAME, + what="The local-authority stage", + outputs=UK_LOCAL_AUTHORITY_OUTPUT_COLUMNS, + operations=local_authority_operation_parameters(), + artifacts=((LOCAL_AUTHORITY_ARTIFACT_ROLE, UK_LOCAL_AUTHORITY_NAMES_RESOURCE),), + ) + + +def uk_local_authorities(codes: pd.Series) -> pd.Series: + """The engine's ``local_authority`` member for each local-authority code. + + Every code must be on the April 2023 roster the committed names resource + carries; a missing or off-roster code is refused before the shared + resolver runs, by a text that names no code. Returns the member names in + the graph's string dtype on ``codes``' index, as the dense line wrote them. + """ + + roster = local_authority_engine_key_by_code() + on_roster = codes.isin(list(roster)).to_numpy(dtype=bool) & codes.notna().to_numpy( + dtype=bool + ) + if not on_roster.all(): + raise ValueError( + f"{UK_LOCAL_AUTHORITY_CODE_COLUMN} holds codes off the April 2023 local " + f"authority roster {UK_LOCAL_AUTHORITY_NAMES_RESOURCE} carries; the engine " + f"input {UK_LOCAL_AUTHORITY_INPUT_COLUMN} is never defaulted " + "(microcosm#953)." + ) + keys = resolve_local_authority_engine_keys(codes.to_numpy(dtype=object)) + return pd.Series(keys, index=codes.index, dtype=_STRING) + + +@dataclass(frozen=True) +class UKLocalAuthorityResult: + """G3's frame and what its evidence reads of it; the source count is + exact and prints only rounded.""" + + frame: Frame = field(repr=False) + clone_count: int + source_households: int = field(repr=False) + + def evidence(self) -> dict[str, object]: + return { + "stage": LOCAL_AUTHORITY_STAGE_NAME, + "clone_count": int(self.clone_count), + "source_households": rounded_count(self.source_households), + "local_authority_vintage": UK_LOCAL_AUTHORITY_VINTAGE, + "names_source_sha256": LAD23_NAMES_SHA256, + "output": UK_LOCAL_AUTHORITY_INPUT_COLUMN, + } + + +def resolve_uk_spine_local_authorities(frame: Frame) -> UKLocalAuthorityResult: + """``frame`` with every household's ``local_authority`` (G3). + + Refused, by name: a frame already carrying the column, a frame without + the drawn code, and a code off the roster (:func:`uk_local_authorities`). + """ + + if not isinstance(frame, Frame): + raise TypeError("The local-authority stage reads the households of a Frame.") + validate_uk_national_frame(frame) + household = frame.table("household") + _refuse_written( + household, UK_LOCAL_AUTHORITY_OUTPUT_COLUMNS, "the local-authority stage" + ) + _require_columns( + household, (UK_LOCAL_AUTHORITY_CODE_COLUMN,), "The local-authority stage" + ) + resolved = household.copy() + resolved[UK_LOCAL_AUTHORITY_INPUT_COLUMN] = uk_local_authorities( + household[UK_LOCAL_AUTHORITY_CODE_COLUMN] + ) + result = _with_household(frame, resolved) + sources = record_source_households( + result, "household", stage=LOCAL_AUTHORITY_STAGE_NAME + ) + return UKLocalAuthorityResult( + frame=result, + clone_count=geographic_clone_count(household), + source_households=distinct_carriers(sources.tolist()), + ) + + +@dataclass(frozen=True) +class UKLocalAuthorityStageTransform: + """Whole-stage transform for the local authority (G3). + + Constructing it checks the stage it runs as; running it calls + :func:`resolve_uk_spine_local_authorities`, which reads the committed + names roster. + """ + + stage: SourceStageSpec + last_result: UKLocalAuthorityResult | None = field( + default=None, init=False, repr=False + ) + + def __post_init__(self) -> None: + _assert_local_authority_stage(self.stage) + + def __call__(self, frame: Frame) -> Frame: + _assert_local_authority_stage(self.stage) + result = resolve_uk_spine_local_authorities(frame) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_LOCAL_AUTHORITY_OUTPUT_COLUMNS + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +# --- G4, area_brma ---------------------------------------------------------------- + +AREA_BRMA_STAGE_NAME = "area_brma" +AREA_BRMA_OPERATION_KIND = "assign_area_brma" +#: The stage's two artifacts, whose bytes enter its contract: the committed +#: area-to-BRMA layer Great Britain reads, and the census counts Northern +#: Ireland draws from, under the role ``frs_brma`` declares them by. +BRMA_LAYER_ARTIFACT_ROLE = "brma_area_layer" +BRMA_LAYER_RESOURCE = "brma_area_layer.json" +BRMA_COUNT_ARTIFACT_ROLE = "count_resource" +UK_AREA_BRMA_OUTPUT_COLUMNS = FRS_BRMA_OUTPUT_COLUMNS +_BRMA_COLUMN = FRS_BRMA_OUTPUT_COLUMNS[0] +_PREDICTORS_OPERATION_KIND = "materialize_rules_engine_predictors" +#: The FRS region of Northern Ireland, the one region of its system +#: (``atomic_area_support``'s profile; a test holds the spelling equal). +_NORTHERN_IRELAND_REGION = "NORTHERN_IRELAND" +#: The household columns the BRMA lookup reads: the drawn system, the two +#: area codes the layer is keyed by and the FRS region. +_BRMA_INPUT_COLUMNS = ( + ASSIGNMENT_COLUMNS[1], + *dict.fromkeys(spec.key for spec in BRMA_GRAINS.values()), + REGION_COLUMN, +) + + +def _brma_draw_parameters() -> dict[str, object]: + """``frs_brma``'s count-table draw, which Northern Ireland keeps.""" + + return { + "count_resource": UK_BRMA_COUNT_RESOURCE, + "count_table_systems": [NI_SYSTEM], + "cell_columns": [REGION_COLUMN, UK_BRMA_PREDICTORS[0]], + "assignment_grain": "benunit", + "collapse_to": "household", + "collapse_method": "uniform_member_pick", + "seed": UK_BRMA_DECLARED_SEEDS[_BRMA_COLUMN], + "salts": { + "benunit": UK_BRMA_BENUNIT_SALT, + "household_pick": UK_BRMA_HOUSEHOLD_PICK_SALT, + }, + } + + +def area_brma_operation_parameters() -> tuple[tuple[str, dict[str, object]], ...]: + """The reviewed operations G4's stage must declare verbatim: the engine + predictor it consumes, then the BRMA it writes, from the layer in Great + Britain (keyed by each system's area code) and from ``frs_brma``'s + count-table draw in Northern Ireland, with that draw's seed and salts.""" + + return ( + ( + _PREDICTORS_OPERATION_KIND, + {"predictors": list(UK_BRMA_PREDICTORS), "consumed_only": True}, + ), + ( + AREA_BRMA_OPERATION_KIND, + { + "output": _BRMA_COLUMN, + "area_system": ASSIGNMENT_COLUMNS[1], + "layer_resource": BRMA_LAYER_RESOURCE, + "method": BRMA_METHOD_VERSION, + "area_codes": {spec.system: spec.key for spec in BRMA_GRAINS.values()}, + **_brma_draw_parameters(), + }, + ), + ) + + +def _brma_artifacts() -> tuple[tuple[str, str], ...]: + return ( + (BRMA_LAYER_ARTIFACT_ROLE, BRMA_LAYER_RESOURCE), + (BRMA_COUNT_ARTIFACT_ROLE, UK_BRMA_COUNT_RESOURCE), + ) + + +def _assert_area_brma_stage(stage: SourceStageSpec) -> None: + _assert_stage( + stage, + name=AREA_BRMA_STAGE_NAME, + what="The BRMA stage", + outputs=UK_AREA_BRMA_OUTPUT_COLUMNS, + operations=area_brma_operation_parameters(), + artifacts=_brma_artifacts(), + ) + + +def load_committed_brma_layer() -> tuple[BRMAAreaLayer, str]: + """The committed area-to-BRMA layer and its bytes' SHA-256.""" + + payload = files("microcosm.build.uk").joinpath(BRMA_LAYER_RESOURCE).read_bytes() + return load_brma_area_layer(payload), hashlib.sha256(payload).hexdigest() + + +def load_committed_brma_counts() -> tuple[Mapping[str, Any], str]: + """The committed census count table ``frs_brma`` draws from, and its + bytes' SHA-256.""" + + payload = files("microcosm.build.uk").joinpath(UK_BRMA_COUNT_RESOURCE).read_bytes() + return json.loads(payload), hashlib.sha256(payload).hexdigest() + + +def _brma_systems(household: pd.DataFrame, what: str) -> np.ndarray: + """Each row's drawn area system, every one of the three, Northern + Ireland's rows in its region and no other row there.""" + + _require_columns(household, _BRMA_INPUT_COLUMNS, what) + systems = household[ASSIGNMENT_COLUMNS[1]].to_numpy(dtype=object) + if not np.isin(systems, list(SYSTEMS)).all(): + raise ValueError( + f"{what} reads a BRMA for households of the systems {list(SYSTEMS)}; " + f"{ASSIGNMENT_COLUMNS[1]} holds another value." + ) + northern_ireland = systems == NI_SYSTEM + in_region = ( + household[REGION_COLUMN].to_numpy(dtype=object) == _NORTHERN_IRELAND_REGION + ) + if not np.array_equal(northern_ireland, in_region): + raise ValueError( + f"{what} draws {NI_SYSTEM} households from the count table of their " + f"region, {_NORTHERN_IRELAND_REGION!r}, alone; the drawn system and the " + "FRS region disagree." + ) + return systems + + +def _check_layer_areas( + household: pd.DataFrame, + systems: np.ndarray, + *, + layer: BRMAAreaLayer, + what: str, +) -> None: + """Every Great Britain row's area is on the layer, in the household's own + FRS region; refused by texts that name no code, so ``area_brmas``, which + reads the layer after this check, never reaches its own refusals.""" + + regions = household[REGION_COLUMN].to_numpy(dtype=object) + for grain, spec in BRMA_GRAINS.items(): + rows = systems == spec.system + if not rows.any(): + continue + codes = household.loc[rows, spec.key].to_numpy(dtype=object) + lookup = layer.brma_by_area[grain] + if not all(isinstance(code, str) and code in lookup for code in codes): + raise ValueError( + f"{what} reads each {spec.system} household's BRMA off its " + f"{spec.key} through {BRMA_LAYER_RESOURCE}; some drawn areas are not " + "on the layer, so the layer and the supports must carry one code " + "vintage (microcosm#931)." + ) + placed = np.asarray( + [layer.region_by_area[grain][code] for code in codes], dtype=object + ) + if not np.array_equal(placed, regions[rows]): + raise ValueError( + f"{what} reads each {spec.system} household's BRMA off its " + f"{spec.key}; {BRMA_LAYER_RESOURCE} places some drawn areas in " + "another FRS region than their households'." + ) + + +def _layer_brmas( + household: pd.DataFrame, + systems: np.ndarray, + current: np.ndarray, + *, + layer: BRMAAreaLayer, + what: str, +) -> np.ndarray: + """``current`` with each Great Britain row's BRMA read off its area + through ``area_brmas``, after :func:`_check_layer_areas`.""" + + _check_layer_areas(household, systems, layer=layer, what=what) + brmas, _ = area_brmas( + layer=layer, + area_system=systems, + lsoa_code=household["lsoa_code"].to_numpy(dtype=object), + oa_code=household["oa_code"].to_numpy(dtype=object), + region=household[REGION_COLUMN].to_numpy(dtype=object), + brma=current, + ) + return np.asarray(brmas, dtype=object) + + +def uk_area_brmas( + person: pd.DataFrame, + benunit: pd.DataFrame, + household: pd.DataFrame, + *, + lha_category: Sequence[object], + layer: BRMAAreaLayer, + count_resource: Mapping[str, Any], + what: str = "The BRMA stage", +) -> np.ndarray: + """Each of ``household``'s rows' BRMA, as an object array in row order. + + A Great Britain row reads its BRMA off its drawn LSOA (England and + Wales) or output area (Scotland) through ``layer``, refusing an area the + layer does not hold or places in another FRS region. A Northern Ireland + row's BRMA is drawn on its own benefit units from ``count_resource`` with + ``frs_brma``'s keys, seed and salts + (:func:`~microcosm.build.uk_runtime.frs_brma.draw_household_brmas`); + ``lha_category`` is aligned to ``benunit``'s rows, and only Northern + Ireland's are read. Refused, too: a system outside the three, and a + Northern Ireland row outside its region or another system's row inside it. + """ + + systems = _brma_systems(household, what) + current = np.full(len(household), None, dtype=object) + northern_ireland = systems == NI_SYSTEM + if northern_ireland.any(): + current[northern_ireland] = draw_household_brmas( + person, + benunit, + household.loc[northern_ireland], + lha_category=lha_category, + count_resource=count_resource, + seed=UK_BRMA_DECLARED_SEEDS[_BRMA_COLUMN], + ) + return _layer_brmas(household, systems, current, layer=layer, what=what) + + +@dataclass(frozen=True) +class UKAreaBRMAResult: + """G4's frame and what its evidence reads of it; the source counts are + exact and print only rounded.""" + + frame: Frame = field(repr=False) + clone_count: int + layer_sha256: str + method: str + count_resource_sha256: str + by_grain: Mapping[str, int] = field(repr=False) + northern_ireland_drawn: int = field(repr=False) + + def evidence(self) -> dict[str, object]: + draw = _brma_draw_parameters() + return { + "stage": AREA_BRMA_STAGE_NAME, + "clone_count": int(self.clone_count), + "source_households_by_grain": { + grain: rounded_count(count) for grain, count in self.by_grain.items() + }, + "northern_ireland_source_households_drawn": rounded_count( + self.northern_ireland_drawn + ), + "layer_sha256": self.layer_sha256, + "method": self.method, + "count_resource_sha256": self.count_resource_sha256, + "seed": draw["seed"], + "salts": draw["salts"], + "predictors": list(UK_BRMA_PREDICTORS), + "output": _BRMA_COLUMN, + } + + +def assign_uk_spine_brmas( + frame: Frame, + *, + lha_category: Sequence[object], + layer: BRMAAreaLayer, + layer_sha256: str, + count_resource: Mapping[str, Any], + count_resource_sha256: str, +) -> UKAreaBRMAResult: + """``frame`` with every household's ``brma`` (G4), as an object column. + + ``lha_category`` is the engine's, aligned to the frame's benefit units. + The evidence counts, over every clone, the distinct source households + whose BRMA each Great Britain grain reads and those Northern Ireland + draws: sources, not benefit units. Refused, by name: a frame already + carrying ``brma`` or without the drawn columns, a misaligned + ``lha_category``, and every refusal of :func:`uk_area_brmas`. + """ + + if not isinstance(frame, Frame): + raise TypeError("The BRMA stage places the households of a Frame.") + validate_uk_national_frame(frame) + household = frame.table("household") + _refuse_written(household, UK_AREA_BRMA_OUTPUT_COLUMNS, "the BRMA stage") + benunit = frame.table("benunit") + if len(lha_category) != len(benunit): + raise ValueError( + f"The BRMA stage reads {UK_BRMA_PREDICTORS[0]} for every benefit unit; " + "the engine's values are not aligned to the frame's benefit units." + ) + brmas = uk_area_brmas( + frame.table("person"), + benunit, + household, + lha_category=lha_category, + layer=layer, + count_resource=count_resource, + ) + placed = household.copy() + placed[_BRMA_COLUMN] = pd.Series(brmas, index=household.index, dtype=object) + result = _with_household(frame, placed) + sources = record_source_households(result, "household", stage=AREA_BRMA_STAGE_NAME) + systems = household[ASSIGNMENT_COLUMNS[1]].to_numpy(dtype=object) + return UKAreaBRMAResult( + frame=result, + clone_count=geographic_clone_count(household), + layer_sha256=layer_sha256, + method=layer.method, + count_resource_sha256=count_resource_sha256, + by_grain=MappingProxyType( + { + grain: distinct_carriers(sources[systems == spec.system].tolist()) + for grain, spec in BRMA_GRAINS.items() + } + ), + northern_ireland_drawn=distinct_carriers( + sources[systems == NI_SYSTEM].tolist() + ), + ) + + +@dataclass(frozen=True) +class UKAreaBRMAStageTransform: + """Whole-stage transform for the BRMA (G4), in ``frs_brma``'s slot. + + ``engine`` is the UK rules engine. Constructing the transform checks the + stage it runs as and loads the committed layer and count table; running + it makes one engine call over the whole frame for ``LHA_category``, as + ``frs_brma`` does, then calls :func:`assign_uk_spine_brmas`. + """ + + stage: SourceStageSpec + engine: object = field(repr=False) + layer: BRMAAreaLayer = field(init=False, repr=False, compare=False) + layer_sha256: str = field(init=False, compare=False) + count_resource: Mapping[str, Any] = field(init=False, repr=False, compare=False) + count_resource_sha256: str = field(init=False, compare=False) + last_result: UKAreaBRMAResult | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + _assert_area_brma_stage(self.stage) + layer, layer_sha256 = load_committed_brma_layer() + counts, counts_sha256 = load_committed_brma_counts() + object.__setattr__(self, "layer", layer) + object.__setattr__(self, "layer_sha256", layer_sha256) + object.__setattr__(self, "count_resource", MappingProxyType(dict(counts))) + object.__setattr__(self, "count_resource_sha256", counts_sha256) + + def __call__(self, frame: Frame) -> Frame: + _assert_area_brma_stage(self.stage) + assert_rules_engine_country(self.engine, "uk") + if not isinstance(frame, Frame): + raise TypeError("The BRMA stage places the households of a Frame.") + # The frame's own refusals come before the engine call they would + # otherwise follow. + household = frame.table("household") + _refuse_written(household, UK_AREA_BRMA_OUTPUT_COLUMNS, "the BRMA stage") + _check_layer_areas( + household, + _brma_systems(household, "The BRMA stage"), + layer=self.layer, + what="The BRMA stage", + ) + materialized = self.engine.materialize( + frame, UK_BRMA_PREDICTORS, uk_time_period(frame) + ) + result = assign_uk_spine_brmas( + frame, + lha_category=materialized[UK_BRMA_PREDICTORS[0]], + layer=self.layer, + layer_sha256=self.layer_sha256, + count_resource=self.count_resource, + count_resource_sha256=self.count_resource_sha256, + ) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_AREA_BRMA_OUTPUT_COLUMNS + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +# --- G5, rfdr --------------------------------------------------------------------- + +RFDR_STAGE_NAME = "rfdr" +RFDR_OPERATION_KIND = "assign_rfdr_flags" +#: The stage's one artifact: the committed RFDR layer, whose bytes enter the +#: stage's contract. +RFDR_ARTIFACT_ROLE = "rfdr_area_layer" +RFDR_LAYER_RESOURCE = "rfdr_area_layer.json" +UK_RFDR_OUTPUT_COLUMNS = (RFDR_COLUMN,) + + +def rfdr_operation_parameters() -> tuple[tuple[str, dict[str, object]], ...]: + """The reviewed operation G5's stage must declare verbatim: the layer and + its method, the drawn columns it reads, the supports whose areas are its + roster, and that a system with no flagged household is reported, not + refused (C-6).""" + + return ( + ( + RFDR_OPERATION_KIND, + { + "layer_resource": RFDR_LAYER_RESOURCE, + "method": RFDR_METHOD_VERSION, + "area_system": ASSIGNMENT_COLUMNS[1], + "area_code": ASSIGNMENT_COLUMNS[0], + "roster_sources": { + system: CONDITIONED_SOURCES[system] for system in RFDR_SYSTEMS + }, + "require_flagged": False, + }, + ), + ) + + +def _assert_rfdr_stage(stage: SourceStageSpec) -> None: + _assert_stage( + stage, + name=RFDR_STAGE_NAME, + what="The fuel-relief stage", + outputs=UK_RFDR_OUTPUT_COLUMNS, + operations=rfdr_operation_parameters(), + artifacts=((RFDR_ARTIFACT_ROLE, RFDR_LAYER_RESOURCE),), + ) + + +def load_committed_rfdr_layer() -> tuple[RFDRAreaLayer, str]: + """The committed RFDR layer and its bytes' SHA-256.""" + + payload = files("microcosm.build.uk").joinpath(RFDR_LAYER_RESOURCE).read_bytes() + return load_rfdr_area_layer(payload), hashlib.sha256(payload).hexdigest() + + +def uk_rfdr_flags( + household: pd.DataFrame, + *, + layer: RFDRAreaLayer, + supports: Mapping[str, AtomicSupport], +) -> np.ndarray: + """Each household row's fuel-relief flag from its drawn output area. + + ``area_flags`` with the support areas of each Great Britain system as the + layer's roster and no zero-flag refusal: a layer code outside its + system's support is refused, a malformed drawn code is refused, and a + system none of whose households is flagged is not. + """ + + _require_columns(household, ASSIGNMENT_COLUMNS[:2], "The fuel-relief stage") + missing = [system for system in RFDR_SYSTEMS if system not in supports] + if missing: + raise ValueError( + f"The fuel-relief stage reads the support areas of {list(RFDR_SYSTEMS)} " + f"as the layer's roster; it lacks {missing}." + ) + flags, _ = area_flags( + layer=layer, + area_system=household[ASSIGNMENT_COLUMNS[1]].to_numpy(dtype=object), + area_code=household[ASSIGNMENT_COLUMNS[0]].to_numpy(dtype=object), + require_flagged=False, + roster={system: supports[system].arrays["area"] for system in RFDR_SYSTEMS}, + ) + return flags + + +@dataclass(frozen=True) +class UKRFDRResult: + """G5's frame and what its evidence reads of it, counted among clone-0 + rows; the source counts are exact and print only rounded.""" + + frame: Frame = field(repr=False) + clone_count: int + layer_sha256: str + method: str + support_sha256: Mapping[str, str] + flagged_by_system: Mapping[str, int] = field(repr=False) + no_flagged_household: Mapping[str, bool] + + def evidence(self) -> dict[str, object]: + # A result exists only for a layer whose codes are all on the roster. + return { + "stage": RFDR_STAGE_NAME, + "clone_count": int(self.clone_count), + "counted_among": "clone_0", + "flagged_source_households": { + system: rounded_count(count) + for system, count in self.flagged_by_system.items() + }, + "no_flagged_household": dict(self.no_flagged_household), + "roster": "pass", + "require_flagged": False, + "layer_sha256": self.layer_sha256, + "method": self.method, + "support_sha256": dict(self.support_sha256), + "output": RFDR_COLUMN, + } + + +def flag_uk_spine_rfdr( + frame: Frame, + *, + layer: RFDRAreaLayer, + layer_sha256: str, + supports: Mapping[str, AtomicSupport], +) -> UKRFDRResult: + """``frame`` with every household's fuel-relief flag (G5). + + The flag is written as a boolean column. The evidence counts, among the + rows of geographic clone 0, the distinct source households each Great + Britain system flags, and whether a system holds households none of which + is flagged, the outcome ``area_flags``' ``require_flagged`` refuses (C-6). + Refused, by name: a frame already carrying the flag or without the drawn + area, and every refusal of :func:`uk_rfdr_flags`. + """ + + if not isinstance(frame, Frame): + raise TypeError("The fuel-relief stage flags the households of a Frame.") + validate_uk_national_frame(frame) + household = frame.table("household") + _refuse_written(household, UK_RFDR_OUTPUT_COLUMNS, "the fuel-relief stage") + flags = uk_rfdr_flags(household, layer=layer, supports=supports) + flagged = household.copy() + flagged[RFDR_COLUMN] = pd.Series(flags, index=household.index, dtype=bool) + result = _with_household(frame, flagged) + sources = record_source_households(result, "household", stage=RFDR_STAGE_NAME) + systems = household[ASSIGNMENT_COLUMNS[1]].to_numpy(dtype=object) + clone_zero = _clone_zero(household) + flagged_by_system, none_flagged = {}, {} + for system in RFDR_SYSTEMS: + rows = clone_zero & (systems == system) + flagged_by_system[system] = distinct_carriers(sources[rows & flags].tolist()) + none_flagged[system] = bool(rows.any() and not (rows & flags).any()) + return UKRFDRResult( + frame=result, + clone_count=geographic_clone_count(household), + layer_sha256=layer_sha256, + method=layer.method, + support_sha256=MappingProxyType( + {system: supports[system].sha256 for system in sorted(supports)} + ), + flagged_by_system=MappingProxyType(flagged_by_system), + no_flagged_household=MappingProxyType(none_flagged), + ) + + +@dataclass(frozen=True) +class UKRFDRStageTransform: + """Whole-stage transform for the fuel-relief flag (G5). + + ``supports`` holds the three conditioned supports' bytes, keyed by + system, each held to canonical bytes; their Great Britain areas are the + layer's roster. Constructing the transform checks the stage it runs as + and loads the committed layer; running it calls + :func:`flag_uk_spine_rfdr`. + """ + + stage: SourceStageSpec + supports: Mapping[str, bytes] = field(repr=False) + decoded_supports: Mapping[str, AtomicSupport] = field( + init=False, repr=False, compare=False + ) + layer: RFDRAreaLayer = field(init=False, repr=False, compare=False) + layer_sha256: str = field(init=False, compare=False) + last_result: UKRFDRResult | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + _assert_rfdr_stage(self.stage) + object.__setattr__( + self, + "decoded_supports", + MappingProxyType(uk_conditioned_supports(self.supports)), + ) + layer, digest = load_committed_rfdr_layer() + object.__setattr__(self, "layer", layer) + object.__setattr__(self, "layer_sha256", digest) + + def __call__(self, frame: Frame) -> Frame: + _assert_rfdr_stage(self.stage) + result = flag_uk_spine_rfdr( + frame, + layer=self.layer, + layer_sha256=self.layer_sha256, + supports=self.decoded_supports, + ) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_RFDR_OUTPUT_COLUMNS + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} + + +# --- The SPI-copy area stage, spi_copy_areas -------------------------------------- + +SPI_COPY_AREAS_STAGE_NAME = "spi_copy_areas" +SPI_COPY_AREAS_OPERATION_KIND = "assign_spi_copy_areas" +#: The columns the stage rewrites on the rows the SPI support channel copied, +#: each in the dtype its column holds: the geography key, G2's three +#: assignment and ten ladder columns, G3's local authority, G4's BRMA and G5's +#: flag. +UK_SPI_COPY_AREAS_REWRITE_COLUMNS = ( + IDENTITY_COLUMN, + *UK_SPINE_AREA_COLUMNS, + UK_LOCAL_AUTHORITY_INPUT_COLUMN, + _BRMA_COLUMN, + RFDR_COLUMN, +) +#: The level every copy is drawn at: its inputs are missing by rule (C-5, +#: C-10), so it draws in its region by households. +_COPY_LEVEL = UK_ATOMIC_FALLBACK_LEVELS[-1].name +#: The two copies the support channel makes, by their ordinal on its branch +#: (the copy-id table's ``spi_support_channel`` field): the SPI copy +#: (``spi_support_channel``) and the income band donor +#: (``spi_income_band_donors``). Spelled here, so that an edit to those +#: stages' modules does not re-key this one; a test holds them equal. +_COPY_KINDS = MappingProxyType({1: "spi_support_copy", 2: "income_band_donor"}) +#: The branches a copy's path may take at this stage's place: the support +#: channel's and the geographic clones'. +_COPY_STAGE_BRANCHES = (SUPPORT_CHANNEL_BRANCH, GEOGRAPHIC_SUPPORT_BRANCH) +#: Which households the copy stage's engine call reads: Northern Ireland's +#: copies alone (C-9). +_ENGINE_HOUSEHOLDS = "northern_ireland_copies" +#: How many simulations that call runs: one, over every clone's Northern +#: Ireland copies together. The scratch frame carries no clone index, so the +#: UK adapter, which batches a call by clone and refuses a frame holding +#: fewer clones than the spine's, takes it as one simulation over the rows it +#: holds, as it does at one clone. +_ENGINE_SIMULATIONS = "one" +#: The outcome the copy stage's evidence gives its draw's validation and the +#: RFDR layer's roster check: passed where it drew copies, not run where the +#: frame holds none. +_CHECKS_PASSED = "pass" +_CHECKS_NOT_RUN = "not_run" +#: The mass record on the engine call's scratch frame, which holds the copies' +#: households alone and is discarded once ``LHA_category`` is read. +_SCRATCH_MASS_REASON = ( + "spi_copy_areas engine call: a scratch subset of the frame holding the " + "Northern Ireland SPI-channel copies' households, read for LHA_category " + "only and discarded" +) + + +def spi_copy_areas_operation_parameters() -> tuple[tuple[str, dict[str, object]], ...]: + """The reviewed operations the copy stage must declare verbatim. + + The engine predictor it consumes, on Northern Ireland's copies alone, in + one simulation over every clone's copies; then which rows it re-places + (the copies the support channel's branch makes, read on the support and + geographic branches), the key it gives them, the area draw (G2's + definition at L3, with G2's stream and seed), the stages whose lookups it + re-runs (G3, G4 with Northern Ireland's count-table draw, seed and salts, + and G5) and the columns it rewrites. + """ + + return ( + ( + _PREDICTORS_OPERATION_KIND, + {"predictors": list(UK_BRMA_PREDICTORS), "consumed_only": True}, + ), + ( + SPI_COPY_AREAS_OPERATION_KIND, + { + "copies": { + "branch": SUPPORT_CHANNEL_BRANCH, + "paths_require": list(_COPY_STAGE_BRANCHES), + "kinds": {str(k): kind for k, kind in _COPY_KINDS.items()}, + }, + "key": { + "column": IDENTITY_COLUMN, + "source": GEOGRAPHIC_IDENTITY_SOURCE, + "vintage_resource": FRS_RELEASE_RESOURCE, + }, + "area_draw": { + "definition_of": ATOMIC_AREA_ASSIGN_STAGE_NAME, + "level": _COPY_LEVEL, + "stream": UK_SPINE_ATOMIC_AREA_STREAM, + "seed": UK_SPINE_ATOMIC_AREA_SEED, + }, + "local_authority": {"as_stage": LOCAL_AUTHORITY_STAGE_NAME}, + "brma": { + "as_stage": AREA_BRMA_STAGE_NAME, + "engine_households": _ENGINE_HOUSEHOLDS, + "engine_simulations": _ENGINE_SIMULATIONS, + **_brma_draw_parameters(), + }, + "rfdr": {"as_stage": RFDR_STAGE_NAME, "require_flagged": False}, + "rewrites": list(UK_SPI_COPY_AREAS_REWRITE_COLUMNS), + }, + ), + ) + + +def _spi_copy_areas_artifacts() -> tuple[tuple[str, str], ...]: + """Every artifact the stages it re-runs read: the FRS release (G1b's + vintage), the names roster (G3), the BRMA layer and counts (G4) and the + RFDR layer (G5).""" + + return ( + (FRS_RELEASE_ARTIFACT_ROLE, FRS_RELEASE_RESOURCE), + (LOCAL_AUTHORITY_ARTIFACT_ROLE, UK_LOCAL_AUTHORITY_NAMES_RESOURCE), + *_brma_artifacts(), + (RFDR_ARTIFACT_ROLE, RFDR_LAYER_RESOURCE), + ) + + +def _assert_spi_copy_areas_stage(stage: SourceStageSpec) -> None: + _assert_stage( + stage, + name=SPI_COPY_AREAS_STAGE_NAME, + what="The SPI-copy area stage", + outputs=(), + rewrites=UK_SPI_COPY_AREAS_REWRITE_COLUMNS, + operations=spi_copy_areas_operation_parameters(), + artifacts=_spi_copy_areas_artifacts(), + ) + + +@dataclass(frozen=True) +class _CopyRows: + """Every row's true source and copy path, which rows the support channel + copied and each row's ordinal on its branch (0 for a row it did not).""" + + sources: np.ndarray + paths: list[tuple[tuple[str, int], ...]] + copies: np.ndarray + ordinals: np.ndarray + + +def _copy_rows(household: pd.DataFrame, what: str) -> _CopyRows: + """The rows the support channel copied, through the shared reader with + the support and geographic branches required, as G1b reads a path.""" + + sources, paths = copy_paths( + household, UK_COPY_LINEAGE, require=_COPY_STAGE_BRANCHES + ) + if any(step not in _COPY_STAGE_BRANCHES for path in paths for step, _ in path): + raise ValueError( + f"{what} re-places the support channel's copies right after its " + f"stages, before any later copy step; a household's path takes a " + f"branch other than {list(_COPY_STAGE_BRANCHES)}." + ) + ordinals = np.asarray( + [dict(path).get(SUPPORT_CHANNEL_BRANCH, 0) for path in paths], dtype=np.int64 + ) + copies = ordinals != 0 + if not np.isin(ordinals[copies], list(_COPY_KINDS)).all(): + raise ValueError( + f"{what} re-places the support channel's SPI copies and income band " + f"donors, its ordinals {list(_COPY_KINDS)}; a household carries " + "another ordinal." + ) + return _CopyRows(sources=sources, paths=paths, copies=copies, ordinals=ordinals) + + +def _copy_keys(household: pd.DataFrame, rows: _CopyRows, what: str) -> list[str]: + """Each copy's own geography key, in row order (:func:`uk_spi_copy_keys`).""" + + _require_columns(household, (SOURCE_YEAR_COLUMN,), what) + copies = rows.copies + years = household[SOURCE_YEAR_COLUMN] + if not pd.api.types.is_integer_dtype(years.dtype) or years.isna().any(): + raise ValueError( + f"{what} reads an integer {SOURCE_YEAR_COLUMN!r}; it holds {years.dtype}." + ) + vintages = frs_vintages_by_source_year() + copy_years = years.to_numpy(dtype=np.int64)[copies] + if not np.isin(copy_years, np.fromiter(vintages, np.int64)).all(): + raise ValueError( + f"{what} keys each copy with the FRS vintage frs_release.json pins for " + f"its {SOURCE_YEAR_COLUMN}; it pins the source year(s) " + f"{sorted(vintages)}, and some copies carry another." + ) + return [ + household_draw_key( + source=GEOGRAPHIC_IDENTITY_SOURCE, + source_vintage=vintages[int(year)], + source_household_id=int(source), + clone_path=path, + ) + for source, year, path in zip( + rows.sources[copies].tolist(), + copy_years.tolist(), + [ + path + for path, copy in zip(rows.paths, copies.tolist(), strict=True) + if copy + ], + strict=True, + ) + ] + + +def uk_spi_copy_keys( + household: pd.DataFrame, *, what: str = "The SPI-copy area stage" +) -> tuple[np.ndarray, list[str]]: + """Which rows the SPI support channel copied, and each one's own key. + + A copy's key is ``household_draw_key`` over the geography identity's + source, the FRS vintage frs_release.json pins for its ``source_year``, + its true source and its copy path read on the support and geographic + branches, as the geography identity keys a household (G1b). So it is the + key a reader of the copy's full path gives it once the later copy steps' + columns exist at ordinal 0, and clone 0's copies keep their keys as K + grows. Returns the copy mask and the copies' keys, in row order. + + Refused, by name: a frame without either branch's columns, malformed + lineage, a path taking a later branch, an ordinal that is neither an SPI + copy's nor a band donor's, a non-integer ``source_year`` and a year the + release pins no vintage for. + """ + + rows = _copy_rows(household, what) + return rows.copies, _copy_keys(household, rows, what) + + +def _rewrite( + column: pd.Series, rows: np.ndarray, values: Sequence[object] +) -> pd.Series: + """``column`` with ``values`` at ``rows``, in the column's own dtype.""" + + data = column.to_numpy(dtype=object).copy() + data[rows] = np.asarray(list(values), dtype=object) + rewritten = pd.Series(data, index=column.index, dtype=column.dtype) + if rewritten.dtype != column.dtype: + raise ValueError( + f"The SPI-copy area stage keeps {column.name!r} in its dtype " + f"{column.dtype}; the rewrite gave {rewritten.dtype}." + ) + return rewritten + + +def _scratch_frame(frame: Frame, household_ids: np.ndarray) -> Frame: + """The frame's rows of ``household_ids``' households, for one engine call. + + Every column and the frame's metadata as the frame holds them, but the + household clone index, with a mass record for the subset's weight, so + the scratch frame is a valid UK frame of its own; it is read for the + engine call and discarded. Without the clone index the UK adapter runs + one simulation over the rows the frame holds, at every clone count: with + it, a spine built with ``clones=K`` would refuse the subset whenever some + clone holds no Northern Ireland copy, since it refuses a call over part + of its pool (:data:`_ENGINE_SIMULATIONS`, C-9). + """ + + person = frame.table("person") + selected = frame.select( + person["person_household_id"].isin(household_ids).to_numpy(dtype=bool) + ) + old_total = float(frame.weights_for("household").total) + new_total = float(selected.weights_for("household").total) + record = MassChangeRecord( + entity="household", + old_total=old_total, + new_total=new_total, + declared_factor=new_total / old_total if old_total else None, + reason=_SCRATCH_MASS_REASON, + ) + weights: Weights = selected.weights_for("household") + tables = {entity: selected.table(entity) for entity in selected.entities} + tables["household"] = tables["household"].drop( + columns=[_CLONE_INDEX_COLUMN], errors="ignore" + ) + scratch = Frame( + tables, + selected.schema, + {"household": weights}, + selected.strata, + mass_log=(*selected.mass_log, record), + metadata=selected.metadata, + ) + validate_uk_national_frame(scratch) + return scratch + + +def _copy_brmas( + frame: Frame, + rows: np.ndarray, + *, + engine: object, + layer: BRMAAreaLayer, + count_resource: Mapping[str, Any], + what: str, +) -> np.ndarray: + """The BRMA of each copy at ``rows`` of ``frame``'s household table. + + ``frame`` holds the copies' new areas. A Great Britain copy reads its + BRMA off its area through the layer; Northern Ireland's copies take one + engine call over their households alone for ``LHA_category``, which reads + only a household's own bedroom entitlement and shared-rate status, one + simulation over every clone's copies (:func:`_scratch_frame`), and then + draw on their own benefit units with ``frs_brma``'s seed and salts, each + copy a draw of its own (C-9). No call runs without such a copy. + """ + + household = frame.table("household") + copies = household.loc[rows] + systems = _brma_systems(copies, what) + # Great Britain's areas are checked before the engine call. + _check_layer_areas(copies, systems, layer=layer, what=what) + current = np.full(len(copies), None, dtype=object) + northern_ireland = systems == NI_SYSTEM + if northern_ireland.any(): + ids = copies.loc[northern_ireland, "household_id"].to_numpy() + scratch = _scratch_frame(frame, ids) + materialized = engine.materialize( + scratch, UK_BRMA_PREDICTORS, uk_time_period(frame) + ) + lha_category = materialized[UK_BRMA_PREDICTORS[0]] + scratch_benunit = scratch.table("benunit") + if len(lha_category) != len(scratch_benunit): + raise ValueError( + f"{what} reads {UK_BRMA_PREDICTORS[0]} for the Northern Ireland " + "copies' benefit units; the engine's values are not aligned to them." + ) + scratch_household = scratch.table("household") + drawn = pd.Series( + draw_household_brmas( + scratch.table("person"), + scratch_benunit, + scratch_household, + lha_category=lha_category, + count_resource=count_resource, + seed=UK_BRMA_DECLARED_SEEDS[_BRMA_COLUMN], + ), + index=scratch_household["household_id"].to_numpy(), + dtype=object, + ) + if not np.isin(ids, drawn.index.to_numpy()).all(): + raise ValueError( + f"{what} draws each Northern Ireland copy's BRMA on the engine " + "call's frame of their households; that frame lacks some of them." + ) + current[northern_ireland] = drawn.reindex(ids).to_numpy(dtype=object) + return _layer_brmas(copies, systems, current, layer=layer, what=what) + + +@dataclass(frozen=True) +class UKSPICopyAreasResult: + """The copy stage's frame and what its evidence reads of it, counted + among clone-0 rows; the source counts are exact and print only rounded.""" + + frame: Frame = field(repr=False) + clone_count: int + seed: int + stream: tuple[object, ...] + definition_sha256: str + support_sha256: Mapping[str, str] + brma_layer_sha256: str + brma_method: str + count_resource_sha256: str + rfdr_layer_sha256: str + rfdr_method: str + by_kind: Mapping[str, Mapping[str, int]] = field(repr=False) + flagged_by_system: Mapping[str, int] = field(repr=False) + no_flagged_copy: Mapping[str, bool] + #: "pass" where the copies were drawn, "not_run" for a frame without a + #: copy, where neither the draw's validation nor the RFDR layer's roster + #: check runs. + checks: str + + def evidence(self) -> dict[str, object]: + # A result with copies exists only where every copy's draw validated + # and the RFDR layer's codes are all on the roster; without a copy + # neither check runs, and both read not_run. + draw = _brma_draw_parameters() + return { + "stage": SPI_COPY_AREAS_STAGE_NAME, + "clone_count": int(self.clone_count), + "counted_among": "clone_0", + "copy_source_households": { + kind: {system: rounded_count(count) for system, count in counts.items()} + for kind, counts in self.by_kind.items() + }, + "flagged_copy_source_households": { + system: rounded_count(count) + for system, count in self.flagged_by_system.items() + }, + "no_flagged_copy": dict(self.no_flagged_copy), + "level": _COPY_LEVEL, + "seed": int(self.seed), + "stream": list(self.stream), + "definition_sha256": self.definition_sha256, + "support_sha256": dict(self.support_sha256), + "validation": {"outcome": self.checks, "scope": _VALIDATION_SCOPE}, + "local_authority_vintage": UK_LOCAL_AUTHORITY_VINTAGE, + "names_source_sha256": LAD23_NAMES_SHA256, + "brma": { + "layer_sha256": self.brma_layer_sha256, + "method": self.brma_method, + "count_resource_sha256": self.count_resource_sha256, + "engine_households": _ENGINE_HOUSEHOLDS, + "engine_simulations": _ENGINE_SIMULATIONS, + "seed": draw["seed"], + "salts": draw["salts"], + }, + "rfdr": { + "roster": self.checks, + "require_flagged": False, + "layer_sha256": self.rfdr_layer_sha256, + "method": self.rfdr_method, + }, + "rewrites": list(UK_SPI_COPY_AREAS_REWRITE_COLUMNS), + } + + +def _definition_sha256(definition: Mapping[str, object]) -> str: + """The definition's digest, as the shared operator computes it.""" + + return hashlib.sha256( + canonical_json(validate_assignment_spec(definition)) + ).hexdigest() + + +def assign_uk_spi_copy_areas( + frame: Frame, + *, + definition: Mapping[str, object], + supports: Mapping[str, AtomicSupport], + engine: object, + brma_layer: BRMAAreaLayer, + brma_layer_sha256: str, + count_resource: Mapping[str, Any], + count_resource_sha256: str, + rfdr_layer: RFDRAreaLayer, + rfdr_layer_sha256: str, +) -> UKSPICopyAreasResult: + """``frame`` with every SPI-channel copy re-keyed and re-placed. + + On the rows the SPI support channel copied (an SPI copy or an income band + donor, :func:`uk_spi_copy_keys`), in turn: each copy's own geography key; + its area, drawn by G2's ``definition`` on a narrow frame of the copies + (their inputs are missing by rule, so every copy draws at L3), derived + and validated; its local authority (G3); its BRMA (G4, Northern Ireland's + copies drawn on their own benefit units after one engine call, a single + simulation, over their households alone); and its fuel-relief flag (G5). + Here G4 runs before G5, the reverse of their order on the spine; neither + reads the other's column. Each rewrite keeps its column's dtype, and + every other row and column is left as it was. The evidence counts, among + clone-0 rows, the distinct source households copied, by kind and system, + and those whose copies are flagged, by Great Britain system; on a frame + without a copy it says the validation and the roster check did not run. + + Refused, naming columns and declared constants only: a frame without a + column the stage rewrites or reads, every refusal of + :func:`uk_spi_copy_keys`, repeated keys, a copy drawn at any level but + L3, engine values not aligned to the Northern Ireland copies' benefit + units or a scratch frame without one of their households, and every + refusal of :func:`draw_uk_spine_areas`, + :func:`uk_local_authorities`, :func:`uk_area_brmas` and + :func:`uk_rfdr_flags`. + """ + + what = "The SPI-copy area stage" + if not isinstance(frame, Frame): + raise TypeError(f"{what} re-places the households of a Frame.") + validate_uk_national_frame(frame) + household = frame.table("household") + _require_columns(household, UK_SPI_COPY_AREAS_REWRITE_COLUMNS, what) + lineage = _copy_rows(household, what) + copies = lineage.copies + keys = _copy_keys(household, lineage, what) + rewritten = household.copy() + rewritten[IDENTITY_COLUMN] = _rewrite(household[IDENTITY_COLUMN], copies, keys) + if not rewritten[IDENTITY_COLUMN].is_unique: + raise ValueError( + f"{what} gives every copy a key of its own; the keys repeat, so a " + "copy's source and path repeat another household's (microcosm#931)." + ) + by_kind: dict[str, dict[str, int]] = {} + flagged_by_system: dict[str, int] = {} + no_flagged: dict[str, bool] = {} + if copies.any(): + draw = draw_uk_spine_areas( + rewritten.loc[copies], definition=definition, supports=supports + ) + if not (np.asarray(draw.levels, dtype=object) == _COPY_LEVEL).all(): + raise ValueError( + f"{what} draws every copy at {_COPY_LEVEL}, its region by " + "households; the draw placed some copies at another level." + ) + for column in UK_SPINE_AREA_COLUMNS: + rewritten[column] = _rewrite( + household[column], copies, draw.columns[column].to_numpy(dtype=object) + ) + rewritten[UK_LOCAL_AUTHORITY_INPUT_COLUMN] = _rewrite( + household[UK_LOCAL_AUTHORITY_INPUT_COLUMN], + copies, + uk_local_authorities(draw.columns[UK_LOCAL_AUTHORITY_CODE_COLUMN]).to_numpy( + dtype=object + ), + ) + placed = _with_household(frame, rewritten) + rewritten[_BRMA_COLUMN] = _rewrite( + household[_BRMA_COLUMN], + copies, + _copy_brmas( + placed, + copies, + engine=engine, + layer=brma_layer, + count_resource=count_resource, + what=what, + ), + ) + flags = uk_rfdr_flags( + rewritten.loc[copies], layer=rfdr_layer, supports=supports + ) + rewritten[RFDR_COLUMN] = _rewrite(household[RFDR_COLUMN], copies, flags) + result = _with_household(frame, rewritten) + sources = record_source_households( + result, "household", stage=SPI_COPY_AREAS_STAGE_NAME + ) + zero_copies = _clone_zero(household) & copies + systems = rewritten[ASSIGNMENT_COLUMNS[1]].to_numpy(dtype=object) + flagged = rewritten[RFDR_COLUMN].to_numpy(dtype=bool) + for ordinal, kind in _COPY_KINDS.items(): + rows = zero_copies & (lineage.ordinals == ordinal) + by_kind[kind] = _sources_by(sources[rows], systems[rows], SYSTEMS) + for system in RFDR_SYSTEMS: + rows = zero_copies & (systems == system) + flagged_by_system[system] = distinct_carriers(sources[rows & flagged].tolist()) + no_flagged[system] = bool(rows.any() and not (rows & flagged).any()) + stream = tuple(validate_assignment_spec(definition)["stream"]) + return UKSPICopyAreasResult( + frame=result, + clone_count=geographic_clone_count(household), + seed=int(stream[3]), + stream=stream, + definition_sha256=_definition_sha256(definition), + support_sha256=MappingProxyType( + {system: supports[system].sha256 for system in sorted(supports)} + ), + brma_layer_sha256=brma_layer_sha256, + brma_method=brma_layer.method, + count_resource_sha256=count_resource_sha256, + rfdr_layer_sha256=rfdr_layer_sha256, + rfdr_method=rfdr_layer.method, + by_kind=MappingProxyType( + {kind: MappingProxyType(counts) for kind, counts in by_kind.items()} + ), + flagged_by_system=MappingProxyType(flagged_by_system), + no_flagged_copy=MappingProxyType(no_flagged), + checks=_CHECKS_PASSED if copies.any() else _CHECKS_NOT_RUN, + ) + + +@dataclass(frozen=True) +class UKSPICopyAreasStageTransform: + """Whole-stage transform for the SPI-copy area stage, after + spi_income_band_donors. + + ``engine`` is the UK rules engine and ``supports`` the three conditioned + supports' bytes, keyed by system, each held to canonical bytes, as G2's + and G5's transforms take them. Constructing the transform checks the + stage it runs as, builds G2's definition with G2's seed, and loads the + committed BRMA layer and counts and the RFDR layer; running it calls + :func:`assign_uk_spi_copy_areas`. + """ + + stage: SourceStageSpec + engine: object = field(repr=False) + supports: Mapping[str, bytes] = field(repr=False) + definition: Mapping[str, object] = field(init=False, repr=False, compare=False) + decoded_supports: Mapping[str, AtomicSupport] = field( + init=False, repr=False, compare=False + ) + brma_layer: BRMAAreaLayer = field(init=False, repr=False, compare=False) + brma_layer_sha256: str = field(init=False, compare=False) + count_resource: Mapping[str, Any] = field(init=False, repr=False, compare=False) + count_resource_sha256: str = field(init=False, compare=False) + rfdr_layer: RFDRAreaLayer = field(init=False, repr=False, compare=False) + rfdr_layer_sha256: str = field(init=False, compare=False) + last_result: UKSPICopyAreasResult | None = field( + default=None, init=False, repr=False + ) + + def __post_init__(self) -> None: + _assert_spi_copy_areas_stage(self.stage) + # The definition holds every support to canonical bytes, so the + # shared decoder alone reads them after it. + definition = uk_conditioned_assignment_definition( + self.supports, seed=UK_SPINE_ATOMIC_AREA_SEED + ) + decoded = { + system: decode_atomic_support(self.supports[system]) for system in SYSTEMS + } + brma_layer, brma_layer_sha256 = load_committed_brma_layer() + counts, counts_sha256 = load_committed_brma_counts() + rfdr_layer, rfdr_layer_sha256 = load_committed_rfdr_layer() + for name, value in ( + ("definition", definition), + ("decoded_supports", MappingProxyType(decoded)), + ("brma_layer", brma_layer), + ("brma_layer_sha256", brma_layer_sha256), + ("count_resource", MappingProxyType(dict(counts))), + ("count_resource_sha256", counts_sha256), + ("rfdr_layer", rfdr_layer), + ("rfdr_layer_sha256", rfdr_layer_sha256), + ): + object.__setattr__(self, name, value) + + def __call__(self, frame: Frame) -> Frame: + _assert_spi_copy_areas_stage(self.stage) + assert_rules_engine_country(self.engine, "uk") + result = assign_uk_spi_copy_areas( + frame, + definition=self.definition, + supports=self.decoded_supports, + engine=self.engine, + brma_layer=self.brma_layer, + brma_layer_sha256=self.brma_layer_sha256, + count_resource=self.count_resource, + count_resource_sha256=self.count_resource_sha256, + rfdr_layer=self.rfdr_layer, + rfdr_layer_sha256=self.rfdr_layer_sha256, + ) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return () + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_private_rent.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_private_rent.py new file mode 100644 index 0000000000..6832a032e5 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_private_rent.py @@ -0,0 +1,631 @@ +"""The clone-first spine's private-rent stage, G7 (microcosm#931). + +After the SPI housing shell has drawn every SPI-channel household's tenure, +bedrooms and rent, the spine's ``private_rent`` stage +(:class:`UKPrivateRentStageTransform`) reads each Great Britain private +renter's rent off the list of rents of the BRMA its drawn area sits in, as the +dense line's ``uk.full.private_rent`` node did after geography +(microcosm#1262). +:func:`~microcosm.build.uk_runtime.private_rent_localisation.localise_rents` +keeps the renter's rank among the FRS renters of its FRS region and bedroom +size, reads that rank off the list of its BRMA and size, and scales it per FRS +region so that, on the census private-renter mix of the region's BRMAs, the +region's mean rent is the FRS level. On the spine: + +- **Each row ranks on its own.** The key ``localise_rents`` groups rows by is + each row's ``geography_household_key``, unique per row here: the geography + identity keys every base row and the SPI-copy area stage every SPI-channel + row. So every geographic clone and every SPI copy is a spine household of + its own. The dense node keyed the pool clones of one spine household alike + (``property_localisation.spine_household_keys``). Those keys need the CGT + lineage, which the spine writes only after this stage, and would give one + key to the clones of an SPI copy whose housing the shell drew separately + for each clone. Per-row keys have neither problem (C-14). +- **Tie uniforms.** Each row's tie-break ``v`` is the keyed uniform of + ``(geography_household_key, "rent_rank")`` on the stage's own stream, + ``("sha256-u53-v1", "uk-spine-private-rent-v1", 0, seed)``, with the seed 0 + the stage declares (C-43). The rank ``u = F(x-) + v (F(x) - F(x-))`` places + every row inside its own weight step, tied or not, so two FRS renters at one + rent in one cell, or two clones of one renter, take ranks of their own. + Clone 0's key, and so its ``v``, is the same at every clone count; its rank + and its region's scale pool over clones, so its rent nests approximately. +- **Sources.** ``localise_rents`` counts a region's FRS renters in true source + households (``receipt_counts``), so a renter's clones and copies count once + toward the sampled rung's rule, which ranks and scales a region holding + fewer FRS renters than ``MIN_SAMPLED_REGION_RENTERS`` on Great Britain's. +- **Strictness.** A full-scale spine ranks and scales every region on its own + FRS renters and refuses a region with none; a sampled rung + (``sampled_rung``) takes the relaxation. + +Only ``rent`` is rewritten: reported Housing Benefit, Universal Credit and +mortgage payments stay as reported (María's ruling of 2026-11-06, item 5), +the weights pass through, and +the stage records its household mass receipt under its own reason. Equality +with the retired dense node is not a goal: it ranked the final frame, where +CGT twins and support copies held keys of their own (C-14). + +This module's refusals give columns, dtypes and declared constants only, never +a key, an id, a BRMA or area of a household, a value or a count of households. +It refuses a re-drawn renter whose BRMA has no list before ``localise_rents`` +runs, so none of that function's own refusals names a household's BRMA: they +name at most an FRS region, an LHA category, or a BRMA of the census counts +that has no list. + +The evidence counts, among the rows of geographic clone 0, where a union over +clones could grow with K, the distinct true source households whose rent is +re-drawn, rounded through ``receipt_counts``, and the (region, category) cells +ranked against their region or against Great Britain; with K, the seed, the +stream, the resources' digests, the method and the strictness (C-44). + +No stage declares it yet: the transform checks the stage it runs as against +the reviewed declaration this module states (name, household grain, no +outputs, ``rent`` as its one rewrite, the reviewed operations verbatim and its +two artifacts), which the declaration will carry. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from importlib.resources import files +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import private_rent_localisation as prl +from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + geographic_clone_count, + ladder_clone_index_column, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_household_mass_conservation_receipt, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.receipt_counts import ( + distinct_carriers, + rounded_count, + source_household_ids, +) +from microcosm.frame import Frame, Weights +from microcosm.graph.randomness import keyed_uniform + +__all__ = [ + "COUNTS_ARTIFACT_ROLE", + "COUNTS_RESOURCE", + "PRIVATE_RENT_OPERATION_KIND", + "PRIVATE_RENT_STAGE_NAME", + "QUANTILES_ARTIFACT_ROLE", + "QUANTILES_RESOURCE", + "RENT_RANK_PROCESS", + "UK_PRIVATE_RENT_INPUTS", + "UK_PRIVATE_RENT_MASS_CONSERVATION_REASON", + "UK_PRIVATE_RENT_REWRITES", + "UK_SPINE_PRIVATE_RENT_SEED", + "UK_SPINE_PRIVATE_RENT_STREAM", + "UK_SPINE_PRIVATE_RENT_STREAM_EXPERIMENT", + "UKPrivateRentResources", + "UKPrivateRentResult", + "UKPrivateRentStageTransform", + "UKSpinePrivateRents", + "draw_uk_spine_private_rents", + "load_committed_private_rent_resources", + "load_private_rent_resources", + "localise_uk_spine_private_rents", + "private_rent_operation_parameters", + "uk_spine_rent_tie_uniforms", +] + +PRIVATE_RENT_STAGE_NAME = "private_rent" +PRIVATE_RENT_OPERATION_KIND = "localise_private_rents" +#: The stage's own stream, apart from the dense node's +#: (``private_rent_localisation.STREAM_EXPERIMENT``), and the seed the stage +#: declares (C-43): one constant, so clone 0's tie-breaks nest at every K. +UK_SPINE_PRIVATE_RENT_STREAM_EXPERIMENT = "uk-spine-private-rent-v1" +UK_SPINE_PRIVATE_RENT_SEED = 0 +UK_SPINE_PRIVATE_RENT_STREAM = ( + "sha256-u53-v1", + UK_SPINE_PRIVATE_RENT_STREAM_EXPERIMENT, + 0, + UK_SPINE_PRIVATE_RENT_SEED, +) +#: The draw's process coordinate after the row's key, as the dense node +#: spelled it. +RENT_RANK_PROCESS = "rent_rank" +#: The stage's two artifacts, whose bytes enter its contract: the BRMAs' +#: weekly lists of rents and the census private-rented households by FRS +#: region, BRMA and bedrooms (the count table ``frs_brma`` reads under the +#: same role). +QUANTILES_ARTIFACT_ROLE = "rent_quantiles" +QUANTILES_RESOURCE = "brma_rent_quantiles.json" +COUNTS_ARTIFACT_ROLE = "count_resource" +COUNTS_RESOURCE = "brma_private_rented_households.json" +_ARTIFACTS = ( + (QUANTILES_ARTIFACT_ROLE, QUANTILES_RESOURCE), + (COUNTS_ARTIFACT_ROLE, COUNTS_RESOURCE), +) +_RENT_COLUMN = "rent" +_CHANNEL_COLUMN = "household_support_channel" +#: The household columns the stage reads, besides ``source_household_id``, +#: the true source the receipts' reader takes, and the clone index the +#: evidence counts clone 0 by, where the frame carries one. +UK_PRIVATE_RENT_INPUTS = ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + "region", + "tenure_type", + _RENT_COLUMN, + "num_bedrooms", + "brma", + _CHANNEL_COLUMN, +) +UK_PRIVATE_RENT_REWRITES = (_RENT_COLUMN,) +#: The household-mass receipt this stage records (the reviewed +#: ``record_mass_conservation_receipt`` operation repeats it): the terminal +#: family gate requires exactly this reason on a valid mass-conserving record. +UK_PRIVATE_RENT_MASS_CONSERVATION_REASON = ( + "Private rents read off the BRMA lists on the source spine: household " + "weights pass through unchanged and total household mass is conserved." +) +_MASS_GATE_COUPLING = ( + "The terminal family gate requires a valid mass-conserving MassChangeRecord " + "carrying exactly this stage-specific reason." +) +_CLONE_INDEX_COLUMN = ladder_clone_index_column("household") +_FLOAT64 = np.dtype("float64") + + +def private_rent_operation_parameters() -> tuple[tuple[str, dict[str, object]], ...]: + """The reviewed operations G7's stage must declare verbatim. + + The rent re-draw: the rows it re-draws (private tenure, a positive rent, + outside Northern Ireland), the per-row key, tie-break process, stream and + seed, the cells it ranks in, the channel it ranks against, its two + resources and the lists' method, the weekly annualisation, the scale band, + the sampled rung's region rule and the method; then the household mass + receipt. + """ + + return ( + ( + PRIVATE_RENT_OPERATION_KIND, + { + "tenure_type": prl.PRIVATE_TENURE, + "excluded_regions": [prl.NORTHERN_IRELAND], + "identity": GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + "process": RENT_RANK_PROCESS, + "stream": UK_SPINE_PRIVATE_RENT_STREAM_EXPERIMENT, + "seed": UK_SPINE_PRIVATE_RENT_SEED, + "categories": list(prl.CATEGORIES), + "reference_channel": prl.FRS_CHANNEL, + "quantiles_resource": QUANTILES_RESOURCE, + "quantiles_method": prl.QUANTILES_METHOD, + "counts_resource": COUNTS_RESOURCE, + "weeks_in_year": prl.WEEKS_IN_YEAR, + "scale_band": list(prl.SCALE_BAND), + "sampled_region_renters": prl.MIN_SAMPLED_REGION_RENTERS, + "method": prl.METHOD_VERSION, + }, + ), + ( + "record_mass_conservation_receipt", + { + "entity": "household", + "reason": UK_PRIVATE_RENT_MASS_CONSERVATION_REASON, + "declared_factor": 1.0, + "gate_coupling": _MASS_GATE_COUPLING, + }, + ), + ) + + +def _assert_private_rent_stage(stage: SourceStageSpec) -> None: + """The stage the transform runs as: its name, household grain, no outputs + and ``rent`` as its one rewrite, the reviewed operations verbatim, and + exactly its two artifacts, each with ``runtime_sha256_required``.""" + + if not isinstance(stage, SourceStageSpec): + raise TypeError("The private-rent stage runs as a SourceStageSpec stage.") + if stage.stage != PRIVATE_RENT_STAGE_NAME: + raise ValueError( + f"Expected stage {PRIVATE_RENT_STAGE_NAME!r}, got {stage.stage!r}." + ) + if stage.grain != "household": + raise ValueError( + "The private-rent stage rewrites households; grain must be " + f"'household', got {stage.grain!r}." + ) + if tuple(stage.outputs) or tuple(stage.rewrites) != UK_PRIVATE_RENT_REWRITES: + raise ValueError( + "The private-rent stage rewrites exactly the rent; the stage must " + f"declare no outputs and the rewrites {UK_PRIVATE_RENT_REWRITES}, got " + f"{tuple(stage.outputs)} / {tuple(stage.rewrites)}." + ) + _assert_closed_world_operations(stage, private_rent_operation_parameters()) + declared = sorted( + (str(artifact.get("role")), str(artifact.get("resource"))) + for artifact in stage.artifacts + if artifact.get("runtime_sha256_required") is True + ) + if len(stage.artifacts) != len(_ARTIFACTS) or declared != sorted(_ARTIFACTS): + raise ValueError( + f"The private-rent stage reads the artifacts {list(_ARTIFACTS)} (role, " + "resource), each with runtime_sha256_required: true, and no other; the " + f"stage declares {len(stage.artifacts)} artifact(s)." + ) + + +@dataclass(frozen=True) +class UKPrivateRentResources: + """The lists of rents and the census counts, parsed, with their bytes' + SHA-256.""" + + quantiles: prl.RentQuantiles = field(repr=False) + counts: Mapping[tuple[str, str], Mapping[str, float]] = field(repr=False) + quantiles_sha256: str + counts_sha256: str + + +def load_private_rent_resources( + *, quantiles: bytes, counts: bytes +) -> UKPrivateRentResources: + """The two resources from their bytes, through the localiser's loaders.""" + + return UKPrivateRentResources( + quantiles=prl.load_rent_quantiles(quantiles), + counts=MappingProxyType(prl.load_private_renter_counts(counts)), + quantiles_sha256=hashlib.sha256(quantiles).hexdigest(), + counts_sha256=hashlib.sha256(counts).hexdigest(), + ) + + +def load_committed_private_rent_resources() -> UKPrivateRentResources: + """The committed lists of rents and census counts the stage declares.""" + + package = files("microcosm.build.uk") + return load_private_rent_resources( + quantiles=package.joinpath(QUANTILES_RESOURCE).read_bytes(), + counts=package.joinpath(COUNTS_RESOURCE).read_bytes(), + ) + + +def uk_spine_rent_tie_uniforms(keys: Sequence[str]) -> np.ndarray: + """Each key's tie-break uniform on the stage's stream. + + The keyed uniform of ``(key, "rent_rank")`` on + :data:`UK_SPINE_PRIVATE_RENT_STREAM`: a function of the key alone, so a + row's draw does not depend on the other rows, their order or K. + """ + + return keyed_uniform( + stream=UK_SPINE_PRIVATE_RENT_STREAM, + keys=[(str(key), RENT_RANK_PROCESS) for key in keys], + ) + + +def _require_columns(household: pd.DataFrame) -> None: + missing = [ + column for column in UK_PRIVATE_RENT_INPUTS if column not in household.columns + ] + if missing: + raise ValueError( + f"The private-rent stage reads household {missing}, which the earlier " + "spine stages write; the frame lacks them (microcosm#931)." + ) + dtype = household[_RENT_COLUMN].dtype + if dtype != _FLOAT64: + raise ValueError( + f"The private-rent stage rewrites a float64 {_RENT_COLUMN!r}, bit for " + f"bit where it keeps it; the column holds {dtype}." + ) + + +def _redrawn_keys(household: pd.DataFrame, redrawn: np.ndarray) -> list[str]: + """The re-drawn rows' geography keys, each present and each its own.""" + + keys = household[GEOGRAPHY_HOUSEHOLD_KEY_COLUMN].to_numpy(dtype=object)[redrawn] + if not all(isinstance(key, str) and key for key in keys): + raise ValueError( + f"Every re-drawn private renter needs its {GEOGRAPHY_HOUSEHOLD_KEY_COLUMN!r}; " + "the geography identity and the SPI-copy area stage write it " + "(microcosm#931)." + ) + if len(set(keys)) != len(keys): + raise ValueError( + "The private-rent stage keys each re-drawn renter's tie-break on its own " + f"{GEOGRAPHY_HOUSEHOLD_KEY_COLUMN!r}, but re-drawn rows share one: every " + "SPI-channel row needs the key the SPI-copy area stage gives it " + "(microcosm#931)." + ) + return [str(key) for key in keys] + + +def _require_rent_lists( + brmas: np.ndarray, categories: np.ndarray, quantiles: prl.RentQuantiles +) -> None: + """Refuse a re-drawn renter whose BRMA has no list for its category, by a + text that names no BRMA, before ``localise_rents`` would name it.""" + + listed = all( + isinstance(brma, str) and (brma, str(category)) in quantiles.knots + for brma, category in zip(brmas, categories, strict=True) + ) + if not listed: + raise ValueError( + "A re-drawn Great Britain private renter's brma has no list of rents " + f"for its bedroom category in {QUANTILES_RESOURCE}; the BRMA stage " + "places every Great Britain household in a BRMA the lists cover " + "(microcosm#931)." + ) + + +def _clone_zero(household: pd.DataFrame) -> np.ndarray: + """Rows of geographic clone 0: every row of a table without the index.""" + + if _CLONE_INDEX_COLUMN not in household.columns: + return np.ones(len(household), dtype=bool) + return household[_CLONE_INDEX_COLUMN].to_numpy(dtype=np.int64) == 0 + + +def _ranked_cells( + rows: pd.DataFrame, *, counted: np.ndarray, strict: bool +) -> tuple[int, int]: + """Among the ``counted`` re-drawn rows' (region, category) cells, those + ranked against their region and those ranked against Great Britain. + + ``rows`` holds every re-drawn row's region, category, channel, weight and + true source, as ``localise_rents`` holds them. Its routing + (``private_rent_localisation.rent_uniforms``): a sampled rung ranks every + cell of a thin region against Great Britain's FRS renters; any other cell + without an FRS renter of positive weight ranks against its region's. + """ + + thin = prl.sampled_thin_regions(rows, strict=strict) + reference = rows[(rows["channel"] == prl.FRS_CHANNEL) & (rows["weight"] > 0)] + held = set( + zip(reference["region"].tolist(), reference["category"].tolist(), strict=True) + ) + cells = set( + zip( + rows.loc[counted, "region"].tolist(), + rows.loc[counted, "category"].tolist(), + strict=True, + ) + ) + great_britain = sum(1 for region, _ in cells if region in thin) + own_region = sum(1 for cell in cells if cell[0] not in thin and cell not in held) + return own_region, great_britain + + +@dataclass(frozen=True) +class UKSpinePrivateRents: + """The re-draw on one household table: every row's rent, which rows were + re-drawn and their tie-break uniforms, and what the evidence counts among + clone-0 rows. The source count is exact; nothing here prints in the repr + but the cell counts and the strictness.""" + + rent: pd.Series = field(repr=False) + redrawn: np.ndarray = field(repr=False) + tie_uniforms: pd.Series = field(repr=False) + source_households_redrawn: int = field(repr=False) + cells_ranked_against_their_region: int + cells_ranked_against_great_britain: int + strict: bool + + +def draw_uk_spine_private_rents( + household: pd.DataFrame, + weights: Sequence[float] | np.ndarray, + *, + resources: UKPrivateRentResources, + strict: bool, +) -> UKSpinePrivateRents: + """Every household row's rent, Great Britain private renters' re-drawn. + + ``localise_rents`` with each row as its own spine household (its + ``geography_household_key``), its true source household, the tie-break + uniforms :func:`uk_spine_rent_tie_uniforms` gives the re-drawn rows' + keys, and ``strict``. ``weights`` are the rows' household weights, in row + order. Returns the rents on ``household``'s index in its row order, every + row ``localise_rents`` does not re-draw bit for bit. Each row's draw + depends on its own key and the cells' pooled ranks and scales, never on + the table's row order. + + Refused, by name: a missing input column, a rent column that is not + float64, weights that do not align, a re-drawn row without its own key, + a re-drawn row whose BRMA has no list for its category, and every refusal + of ``localise_rents`` and of the receipts' source reader. + """ + + _require_columns(household) + weight = np.asarray(weights, dtype=float) + if weight.shape != (len(household),): + raise ValueError( + "The private-rent stage reads one household weight per household row." + ) + sources = source_household_ids(household, stage="The private-rent stage") + region = household["region"].to_numpy(dtype=object) + tenure = household["tenure_type"].to_numpy(dtype=object) + rent = household[_RENT_COLUMN].to_numpy(dtype=float) + redrawn = prl.redraw_mask(region, tenure, rent) + keys = _redrawn_keys(household, redrawn) + bedrooms = household["num_bedrooms"].to_numpy() + categories = prl.rent_categories(bedrooms[redrawn]) + brma = household["brma"].to_numpy(dtype=object) + _require_rent_lists(brma[redrawn], categories, resources.quantiles) + uniforms = uk_spine_rent_tie_uniforms(keys) + spine_key = np.full(len(household), None, dtype=object) + spine_key[redrawn] = keys + channel = household[_CHANNEL_COLUMN].to_numpy(dtype=object) + new_rent, _ = prl.localise_rents( + spine_key=spine_key, + source=sources, + region=region, + tenure_type=tenure, + rent=rent, + num_bedrooms=bedrooms, + brma=brma, + channel=channel, + weight=weight, + tie_uniform=dict(zip(keys, (float(v) for v in uniforms), strict=True)), + quantiles=resources.quantiles, + counts=resources.counts, + strict=bool(strict), + ) + counted = _clone_zero(household)[redrawn] + rows = pd.DataFrame( + { + "region": region[redrawn].astype(str), + "category": categories, + "channel": channel[redrawn].astype(str), + "weight": weight[redrawn], + "source": sources[redrawn], + } + ) + own_region, great_britain = _ranked_cells(rows, counted=counted, strict=strict) + return UKSpinePrivateRents( + rent=pd.Series(new_rent, index=household.index, name=_RENT_COLUMN), + redrawn=redrawn, + tie_uniforms=pd.Series(uniforms, index=household.index[redrawn], dtype=float), + source_households_redrawn=distinct_carriers(sources[redrawn][counted].tolist()), + cells_ranked_against_their_region=own_region, + cells_ranked_against_great_britain=great_britain, + strict=bool(strict), + ) + + +@dataclass(frozen=True) +class UKPrivateRentResult: + """G7's frame and what its evidence reads of it, counted among clone-0 + rows; the source count is exact and prints only rounded.""" + + frame: Frame = field(repr=False) + clone_count: int + strict: bool + quantiles_sha256: str + counts_sha256: str + source_households_redrawn: int = field(repr=False) + cells_ranked_against_their_region: int + cells_ranked_against_great_britain: int + + def evidence(self) -> dict[str, object]: + return { + "stage": PRIVATE_RENT_STAGE_NAME, + "clone_count": int(self.clone_count), + "counted_among": "clone_0", + "source_households_redrawn": rounded_count(self.source_households_redrawn), + "cells_ranked_against_their_region": int( + self.cells_ranked_against_their_region + ), + "cells_ranked_against_great_britain": int( + self.cells_ranked_against_great_britain + ), + "strict": bool(self.strict), + "seed": UK_SPINE_PRIVATE_RENT_SEED, + "stream": list(UK_SPINE_PRIVATE_RENT_STREAM), + "quantiles_sha256": self.quantiles_sha256, + "counts_sha256": self.counts_sha256, + "method": prl.METHOD_VERSION, + "weights": "unchanged", + "rewrites": list(UK_PRIVATE_RENT_REWRITES), + } + + +def localise_uk_spine_private_rents( + frame: Frame, + *, + resources: UKPrivateRentResources, + sampled_rung: bool = False, +) -> UKPrivateRentResult: + """``frame`` with every Great Britain private renter's rent re-drawn (G7). + + :func:`draw_uk_spine_private_rents` on the frame's households and + weights, strict unless ``sampled_rung``. Rows, ids, weights, every other + column and the metadata are the frame's own; the mass log gains the + stage's receipt. Refused, by name: what that function refuses. + """ + + if not isinstance(frame, Frame): + raise TypeError("The private-rent stage rewrites the households of a Frame.") + if not isinstance(sampled_rung, bool): + raise TypeError("sampled_rung says whether the spine is sampled: a bool.") + validate_uk_national_frame(frame) + household = frame.table("household") + # The household weights pass through unchanged. Annotated so the stage's + # static walk meets microcosm.frame's Weights module. + weights: Weights = frame.weights_for("household") + draw = draw_uk_spine_private_rents( + household, weights.values, resources=resources, strict=not sampled_rung + ) + rewritten = household.copy() + rewritten[_RENT_COLUMN] = draw.rent + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = rewritten + result = Frame( + tables, + frame.schema, + {"household": weights}, + frame.strata, + mass_log=( + *frame.mass_log, + uk_household_mass_conservation_receipt( + frame, UK_PRIVATE_RENT_MASS_CONSERVATION_REASON + ), + ), + metadata=frame.metadata, + ) + validate_uk_national_frame(result) + return UKPrivateRentResult( + frame=result, + clone_count=geographic_clone_count(household), + strict=draw.strict, + quantiles_sha256=resources.quantiles_sha256, + counts_sha256=resources.counts_sha256, + source_households_redrawn=draw.source_households_redrawn, + cells_ranked_against_their_region=draw.cells_ranked_against_their_region, + cells_ranked_against_great_britain=draw.cells_ranked_against_great_britain, + ) + + +@dataclass(frozen=True) +class UKPrivateRentStageTransform: + """Whole-stage transform for the private rents (G7). + + ``sampled_rung`` says whether the spine is sampled: a full-scale spine is + strict. Constructing the transform checks the stage it runs as and loads + the committed lists of rents and census counts; running it calls + :func:`localise_uk_spine_private_rents`. + """ + + stage: SourceStageSpec + sampled_rung: bool = False + resources: UKPrivateRentResources = field(init=False, repr=False, compare=False) + last_result: UKPrivateRentResult | None = field( + default=None, init=False, repr=False + ) + + def __post_init__(self) -> None: + _assert_private_rent_stage(self.stage) + if not isinstance(self.sampled_rung, bool): + raise TypeError("sampled_rung says whether the spine is sampled: a bool.") + object.__setattr__(self, "resources", load_committed_private_rent_resources()) + + def __call__(self, frame: Frame) -> Frame: + _assert_private_rent_stage(self.stage) + result = localise_uk_spine_private_rents( + frame, resources=self.resources, sampled_rung=self.sampled_rung + ) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return () + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_atomic_area_conditioning.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_atomic_area_conditioning.py new file mode 100644 index 0000000000..00cf53ac38 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_atomic_area_conditioning.py @@ -0,0 +1,566 @@ +"""The UK's conditioned area draw: definition, inputs and ladder (microcosm#931). + +The spine's area draw (G2) conditions each household's area on its IMD decile +and its observed council-tax band, falling back along a declared ladder where +a cell has no mass. These tests hold the definition to canonical conditioned +supports, the ten ladder layers and the spine's stream, Northern Ireland to +households; the draw inputs to O14 iv and v, SPI-channel rows to missing +inputs; each household's level to a per-row reading of the toy supports; and +today's region-only definition to its bytes. Toy supports and invented +households only; no refusal prints a key, a code of a household or a count. +""" + +from __future__ import annotations + +import hashlib + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.atomic_geography import ( + assign_atomic, + decode_atomic_support, + encode_atomic_support, +) +from microcosm.build.source_lineage import copy_paths +from microcosm.build.uk_runtime import atomic_area_conditioning as conditioning +from microcosm.build.uk_runtime.atomic_area_conditioning import ( + BAND_INPUT_COLUMN, + DECILE_INPUT_COLUMN, + DEFAULT_WEIGHT_COLUMN, + DRAW_INPUT_COLUMNS, + IDENTITY_COLUMN, + REGION_COLUMN, + UK_ATOMIC_FALLBACK_LEVELS, + UK_DRAW_CONSTRAINTS, + UK_FALLBACK_STATUSES, + UK_SPINE_ATOMIC_AREA_SEED, + UK_SPINE_ATOMIC_AREA_STREAM, + UK_SPINE_LADDER_COLUMNS, + uk_band_weight_map, + uk_conditioned_assignment_definition, + uk_conditioned_supports, + uk_draw_frame, + uk_draw_inputs, + uk_draw_levels, + uk_fallback_reasons, + uk_support_channel_copies, + uk_system_fallback_levels, +) +from microcosm.build.uk_runtime.atomic_area_support import ( + ASSIGNMENT_COLUMNS, + BAND_WEIGHT_COLUMNS, + CONDITIONED_SOURCES, + IMD_DECILE_COLUMN, + IMD_DECILES, + UK_NATIVE_ALIAS_COLUMNS, + uk_atomic_assignment_definition, +) +from microcosm.build.uk_runtime.atomic_area_support import ( + IDENTITY_COLUMN as SUPPORT_IDENTITY_COLUMN, +) +from microcosm.build.uk_runtime.council_tax_constants import UK_COUNCIL_TAX_BANDS +from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE +from microcosm.graph.canonical import canonical_json +from test_support.microcosm_build.uk_atomic_area_support import payloads +from test_support.microcosm_build.uk_atomic_support_fixtures import toy_support_payloads +from test_support.microcosm_build.uk_conditioned_support_fixtures import ( + EW, + NI, + SCOT, + TOY_HOUSEHOLDS, + ToyHousehold, + toy_conditioned_support_payloads, + toy_conditioned_supports, + toy_household_table, +) + +_STRING = pd.StringDtype(storage="python") +#: Today's region-only definition on the invented supports, seed 42: its +#: canonical bytes' SHA-256 at the parent of the conditioned draw. The national +#: problem, the dense measures, the inspection export and the geography +#: identity hash atomic_area_support, so the conditioned draw lives in a +#: module of its own and leaves this definition byte for byte (C-28). +_V1_DEFINITION_SHA256 = ( + "5947915e81ccf8a8fb03f7b856b4309b7d59adac4b86322b07e0d9b2020cdf0d" +) + + +@pytest.fixture(scope="module") +def definition() -> dict: + return uk_conditioned_assignment_definition( + toy_conditioned_support_payloads(), seed=UK_SPINE_ATOMIC_AREA_SEED + ) + + +@pytest.fixture(scope="module") +def supports() -> dict: + return toy_conditioned_supports() + + +def _households(cases=TOY_HOUSEHOLDS, **lineage) -> pd.DataFrame: + """The cases' raw inputs with invented keys and sources, on an index that + is not a range, so the adapter must keep the table's own.""" + + household = toy_household_table(cases) + household.index = pd.Index(np.arange(len(cases)) * 7 + 3, name="row") + household[IDENTITY_COLUMN] = pd.Series( + [f"invented-key-{position}" for position in range(len(cases))], + index=household.index, + dtype=_STRING, + ) + household["source_household_id"] = np.arange(1, len(cases) + 1, dtype=np.int64) + for column, values in lineage.items(): + household[column] = values + return household + + +def _system_rows(definition: dict, system: str) -> dict: + (row,) = [entry for entry in definition["systems"] if entry["id"] == system] + return row + + +# --- The definition ---------------------------------------------------------------- + + +def test_the_definition_declares_the_conditioned_draw(definition) -> None: + assert definition["identity"] == [IDENTITY_COLUMN] + assert IDENTITY_COLUMN == SUPPORT_IDENTITY_COLUMN + assert definition["stream"] == [ + "sha256-u53-v1", + UK_SPINE_ATOMIC_AREA_STREAM, + 0, + UK_SPINE_ATOMIC_AREA_SEED, + ] + assert definition["outputs"] == dict( + zip(("area", "system", "basis"), ASSIGNMENT_COLUMNS, strict=True) + ) + for system, sources in CONDITIONED_SOURCES.items(): + entry = _system_rows(definition, system) + assert entry["source"] == sources + assert entry["observed_area"] is None + assert entry["constraints"] == [ + {"input": REGION_COLUMN, "support": "frs_region", "required": True}, + { + "input": DECILE_INPUT_COLUMN, + "support": IMD_DECILE_COLUMN, + "required": False, + }, + ] + + +def test_great_britain_samples_the_bands_weight_and_northern_ireland_households( + definition, +) -> None: + for system in (EW, SCOT): + (stage,) = _system_rows(definition, system)["stages"] + assert stage == { + "level": "area", + "weight_by": { + "input": BAND_INPUT_COLUMN, + "map": uk_band_weight_map(system), + "default": DEFAULT_WEIGHT_COLUMN, + }, + } + assert sorted(uk_band_weight_map(system).values()) == sorted( + BAND_WEIGHT_COLUMNS[system] + ) + assert sorted(uk_band_weight_map(EW)) == list("ABCDEFGHI") + assert sorted(uk_band_weight_map(SCOT)) == list("ABCDEFGH") + # Domestic rates: no band weights, so no weight_by in Northern Ireland. + assert _system_rows(definition, NI)["stages"] == [ + {"level": "area", "weight": DEFAULT_WEIGHT_COLUMN} + ] + assert uk_band_weight_map(NI) == {} + assert [level.name for level in uk_system_fallback_levels(NI)] == ["L1", "L3"] + assert uk_system_fallback_levels(EW) == UK_ATOMIC_FALLBACK_LEVELS + + +def test_the_layers_are_the_ten_ladder_columns_and_carry_the_supports_metadata( + definition, supports +) -> None: + for entry in definition["systems"]: + outputs = [layer["output"] for layer in entry["layers"]] + assert outputs == list(UK_SPINE_LADDER_COLUMNS) + assert not set(outputs) & set(UK_NATIVE_ALIAS_COLUMNS) + columns = supports[entry["id"]].metadata["columns"] + for layer in entry["layers"]: + assert layer["input"] == ( + "area" if layer["output"] == "oa_code" else layer["output"] + ) + assert {key: layer[key] for key in ("source", "vintage", "relation")} == { + key: columns[layer["input"]][key] + for key in ("source", "vintage", "relation") + } + + +def test_the_definition_takes_canonical_conditioned_bytes_only() -> None: + conditioned = toy_conditioned_support_payloads() + # Today's supports, without the conditioning columns. + with pytest.raises(ValueError, match="conditioned support columns differ"): + uk_conditioned_assignment_definition(toy_support_payloads(), seed=0) + # Self-consistent bytes the adapter would not write: one area's FRS + # region disagrees with its region code. + support = decode_atomic_support(conditioned[EW]) + metadata = { + **dict(support.metadata), + "columns": { + name: dict(column) for name, column in support.metadata["columns"].items() + }, + } + arrays = {name: np.asarray(array).copy() for name, array in support.arrays.items()} + arrays["frs_region"][0] = ( + "WALES" if arrays["frs_region"][0] != "WALES" else "LONDON" + ) + forged = encode_atomic_support(metadata, arrays) + with pytest.raises(ValueError, match="not the canonical conditioned output"): + uk_conditioned_assignment_definition({**conditioned, EW: forged}, seed=0) + with pytest.raises(ValueError, match="three systems"): + uk_conditioned_assignment_definition( + {EW: conditioned[EW], SCOT: conditioned[SCOT]}, seed=0 + ) + with pytest.raises(ValueError, match="immutable support bytes"): + uk_conditioned_assignment_definition( + {**conditioned, NI: bytearray(conditioned[NI])}, seed=0 + ) + for seed in (-1, 2**32, True, 1.0): + with pytest.raises(ValueError, match="seed must be a bounded integer"): + uk_conditioned_assignment_definition(conditioned, seed=seed) + + +def test_the_conditioned_supports_decode_only_canonical_bytes() -> None: + decoded = uk_conditioned_supports(toy_conditioned_support_payloads()) + assert list(decoded) == [EW, SCOT, NI] + assert all( + support.metadata["system"] == system for system, support in decoded.items() + ) + with pytest.raises(ValueError, match="conditioned support columns differ"): + uk_conditioned_supports(toy_support_payloads()) + + +def test_the_region_only_definition_keeps_its_bytes() -> None: + """Today's v1 definition, which the conditioned one leaves as it is.""" + + v1 = uk_atomic_assignment_definition(payloads(), seed=42) + assert hashlib.sha256(canonical_json(v1)).hexdigest() == _V1_DEFINITION_SHA256 + conditioned = uk_conditioned_assignment_definition( + toy_conditioned_support_payloads(), seed=42 + ) + assert v1["stream"][1] != conditioned["stream"][1] + for old, new in zip(v1["systems"], conditioned["systems"], strict=True): + assert {key: old[key] for key in ("id", "level", "vintage", "selector")} == { + key: new[key] for key in ("id", "level", "vintage", "selector") + } + assert old["constraints"] == new["constraints"][:1] + assert old["stages"] == [{"level": "area", "weight": "households"}] + assert {layer["output"] for layer in new["layers"]} < { + layer["output"] for layer in old["layers"] + } + + +def test_the_spellings_follow_their_owners() -> None: + from microcosm.build.uk_runtime.frs_spine import ( + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, + FRS_IMD_DECILE_COLUMN, + FRS_IMD_DECILE_UNOBSERVED, + ) + from microcosm.build.uk_runtime.geography_ladder import UK_GEOGRAPHY_LADDER_COLUMNS + from microcosm.build.uk_runtime.local_authority_input import ( + UK_LOCAL_AUTHORITY_INPUT_COLUMN, + ) + + assert conditioning.COUNCIL_TAX_BAND_OBSERVED_COLUMN == ( + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN + ) + assert conditioning.UNOBSERVED_IMD_DECILE == FRS_IMD_DECILE_UNOBSERVED + assert IMD_DECILE_COLUMN == FRS_IMD_DECILE_COLUMN + assert UK_SPINE_LADDER_COLUMNS == tuple( + column + for column in UK_GEOGRAPHY_LADDER_COLUMNS + if column != UK_LOCAL_AUTHORITY_INPUT_COLUMN + ) + assert set(uk_band_weight_map(EW)) <= set(UK_COUNCIL_TAX_BANDS) + branches = {branch.name for branch in UK_COPY_LINEAGE.branches} + assert conditioning.SUPPORT_CHANNEL_BRANCH in branches + assert dict(UK_DRAW_CONSTRAINTS) == { + REGION_COLUMN: "frs_region", + DECILE_INPUT_COLUMN: IMD_DECILE_COLUMN, + } + + +# --- The draw inputs (O14 iv and v) ------------------------------------------------ + + +def test_the_inputs_read_the_decile_as_text_and_the_band_where_observed() -> None: + household = _households() + inputs = uk_draw_inputs(household) + assert list(inputs.columns) == list(DRAW_INPUT_COLUMNS) + assert inputs.index.equals(household.index) + assert all(inputs[column].dtype == _STRING for column in inputs.columns) + for (_, row), case in zip(inputs.iterrows(), TOY_HOUSEHOLDS, strict=True): + assert row[REGION_COLUMN] == case.region + if case.imd_decile: + assert row[DECILE_INPUT_COLUMN] == str(case.imd_decile) + else: + assert pd.isna(row[DECILE_INPUT_COLUMN]) + if case.council_tax_band_observed: + assert row[BAND_INPUT_COLUMN] == case.council_tax_band + else: + assert pd.isna(row[BAND_INPUT_COLUMN]) + + +def test_every_decile_reads_as_its_own_code_ten_included() -> None: + """The decile reads as the supports' codes "1" to "10", the two-digit one + included, and 0 as missing.""" + + cases = [ToyHousehold("LONDON", decile, "B", True, "L0") for decile in range(0, 11)] + inputs = uk_draw_inputs(_households(cases=cases)) + assert inputs[DECILE_INPUT_COLUMN].tolist()[1:] == list(IMD_DECILES) + assert inputs[DECILE_INPUT_COLUMN].tolist()[-1] == "10" + assert pd.isna(inputs[DECILE_INPUT_COLUMN].iloc[0]) + + +def test_support_channel_copies_draw_without_inputs(definition, supports) -> None: + copies = np.asarray([position % 2 for position in range(len(TOY_HOUSEHOLDS))]) + household = _households( + household_support_channel=np.where(copies == 1, "spi", "frs").astype(object), + household_support_clone_index=copies.astype(np.int64), + ) + assert uk_support_channel_copies(household).tolist() == (copies == 1).tolist() + inputs = uk_draw_inputs(household) + assert ( + inputs.loc[copies == 1, [DECILE_INPUT_COLUMN, BAND_INPUT_COLUMN]] + .isna() + .all() + .all() + ) + levels = uk_draw_levels(inputs, definition, supports) + assert set(levels[copies == 1]) == {"L3"} + assert levels[copies == 0].tolist() == [ + case.level + for case, copy in zip(TOY_HOUSEHOLDS, copies, strict=True) + if not copy + ] + # A table without the channel's columns has no copy. + assert not uk_support_channel_copies(_households()).any() + + +def test_a_later_copy_of_an_spi_copy_is_an_spi_channel_row() -> None: + household = _households( + household_support_channel=np.asarray( + ["spi"] + ["frs"] * (len(TOY_HOUSEHOLDS) - 1), dtype=object + ), + household_support_clone_index=np.asarray( + [1] + [0] * (len(TOY_HOUSEHOLDS) - 1), dtype=np.int64 + ), + household_is_capital_gains_clone=np.asarray( + [True, True] + [False] * (len(TOY_HOUSEHOLDS) - 2) + ), + ) + _, paths = copy_paths(household, UK_COPY_LINEAGE) + assert paths[0] == (("spi_support_channel", 1), ("cgt_incidence_clone", 1)) + assert uk_support_channel_copies(household).tolist()[:3] == [True, False, False] + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda h: h.drop(columns=["council_tax_band_observed"]), "lacks"), + (lambda h: h.assign(**{IDENTITY_COLUMN: pd.NA}), IDENTITY_COLUMN), + (lambda h: h.assign(region=" LONDON"), "region must hold"), + (lambda h: h.assign(imd_decile=11), "outside 0 to 10"), + (lambda h: h.assign(imd_decile=-1), "outside 0 to 10"), + (lambda h: h.assign(imd_decile=h["imd_decile"].astype(float)), "integer"), + (lambda h: h.assign(imd_decile=h["imd_decile"] > 0), "integer"), + ( + lambda h: h.assign( + council_tax_band_observed=h["council_tax_band_observed"] + .astype("boolean") + .where(h.index != h.index[0], pd.NA) + ), + "complete boolean", + ), + (lambda h: h.assign(council_tax_band_observed=1), "complete boolean"), + ( + lambda h: h.assign(council_tax_band="J", council_tax_band_observed=True), + "not a band A to I", + ), + ], +) +def test_malformed_inputs_are_refused_by_column(mutate, message) -> None: + household = mutate(_households()) + with pytest.raises(ValueError, match=message) as refused: + uk_draw_inputs(household) + # Columns and declared constants only: no key and no household's region. + text = str(refused.value) + assert "invented-key" not in text + assert not any(case.region in text for case in TOY_HOUSEHOLDS) + + +# --- The levels --------------------------------------------------------------------- + + +def _first_level_with_mass(support, case) -> str: + """The first level whose inputs hold and whose cell has mass, row by row.""" + + decile = str(case.imd_decile) if case.imd_decile else None + band = case.council_tax_band if case.council_tax_band_observed else None + regions = np.asarray(support.arrays["frs_region"]) + deciles = np.asarray(support.arrays[IMD_DECILE_COLUMN]) + weights = uk_band_weight_map(support.metadata["system"]) + for level in uk_system_fallback_levels(support.metadata["system"]): + keeps = DECILE_INPUT_COLUMN in level.constraints + if keeps and decile is None: + continue + if level.weight_by and band not in weights: + continue + rows = regions == case.region + if keeps: + rows &= deciles == decile + column = weights[band] if level.weight_by else DEFAULT_WEIGHT_COLUMN + if int(np.asarray(support.arrays[column], dtype=np.int64)[rows].sum()) > 0: + return level.name + raise AssertionError("an invented household has no level") + + +def test_each_household_takes_the_first_level_whose_cell_has_mass( + definition, supports +) -> None: + levels = uk_draw_levels(uk_draw_inputs(_households()), definition, supports) + system_of = {"NORTHERN_IRELAND": NI, "SCOTLAND": SCOT} + expected = [ + _first_level_with_mass(supports[system_of.get(case.region, EW)], case) + for case in TOY_HOUSEHOLDS + ] + assert levels.tolist() == expected == [case.level for case in TOY_HOUSEHOLDS] + # Every level is exercised, and Northern Ireland takes L1 or L3 only. + assert set(levels) == {"L0", "L1", "L2", "L3"} + northern_ireland = [c.region == "NORTHERN_IRELAND" for c in TOY_HOUSEHOLDS] + assert set(levels[northern_ireland]) == {"L1", "L3"} + + +def test_the_chooser_reads_its_rule_and_default_from_the_definition( + definition, +) -> None: + # The draw samples a missing band by the stage's default; the chooser's + # default is that same column, and its map the stage's. + for entry in definition["systems"]: + rule, default = conditioning._system_rules(entry) + assert default == DEFAULT_WEIGHT_COLUMN + if entry["id"] == NI: + assert rule is None + else: + assert rule == { + "input": BAND_INPUT_COLUMN, + "map": uk_band_weight_map(entry["id"]), + } + + +def test_the_chooser_is_called_with_the_definitions_own_arguments( + monkeypatch, definition, supports +) -> None: + """Every toy cell with population has households too, so the levels alone + cannot tell the default weight from another: the chooser's arguments are + recorded and held to each system's stage.""" + + calls: list[dict] = [] + real = conditioning.choose_fallback_levels + + def recording(support, households, **arguments): + calls.append({"system": support.metadata["system"], **arguments}) + return real(support, households, **arguments) + + monkeypatch.setattr(conditioning, "choose_fallback_levels", recording) + uk_draw_levels(uk_draw_inputs(_households()), definition, supports) + assert sorted(call["system"] for call in calls) == sorted((EW, SCOT, NI)) + for call in calls: + (entry,) = [e for e in definition["systems"] if e["id"] == call["system"]] + rule, default = conditioning._system_rules(entry) + assert call["default_weight"] == default == DEFAULT_WEIGHT_COLUMN + assert call["weight_by"] == rule + assert call["constraint_columns"] == dict(UK_DRAW_CONSTRAINTS) + assert [level.name for level in call["levels"]] == [ + level.name for level in uk_system_fallback_levels(call["system"]) + ] + + +def test_a_region_no_system_selects_is_refused(definition, supports) -> None: + household = _households().assign(region="UNKNOWN") + with pytest.raises(ValueError, match="exactly one area system"): + uk_draw_levels(uk_draw_inputs(household), definition, supports) + with pytest.raises(ValueError, match="decoded support for every system"): + uk_draw_levels(uk_draw_inputs(_households()), definition, {EW: supports[EW]}) + + +def test_the_draw_frame_drops_what_each_level_does_not_keep( + definition, supports +) -> None: + inputs = uk_draw_inputs(_households()) + levels = uk_draw_levels(inputs, definition, supports) + frame = uk_draw_frame(inputs, levels) + assert all(frame[column].dtype == _STRING for column in frame.columns) + keeps_decile = np.isin(levels, ["L0", "L1"]) + keeps_band = np.isin(levels, ["L0", "L2"]) + assert frame[DECILE_INPUT_COLUMN][~keeps_decile].isna().all() + assert frame[BAND_INPUT_COLUMN][~keeps_band].isna().all() + assert frame[DECILE_INPUT_COLUMN][keeps_decile].equals( + inputs[DECILE_INPUT_COLUMN][keeps_decile] + ) + assert frame[BAND_INPUT_COLUMN][keeps_band].equals( + inputs[BAND_INPUT_COLUMN][keeps_band] + ) + assert frame[[IDENTITY_COLUMN, REGION_COLUMN]].equals( + inputs[[IDENTITY_COLUMN, REGION_COLUMN]] + ) + with pytest.raises(ValueError, match="outside the UK fallback ladder"): + uk_draw_frame(inputs, ["L4"] * len(inputs)) + with pytest.raises(ValueError, match="one level per household"): + uk_draw_frame(inputs, levels[:-1]) + + +def test_one_definition_draws_every_level_inside_its_cell(definition, supports) -> None: + """A level's draw keeps its own inputs alone, and lands where it has mass.""" + + inputs = uk_draw_inputs(_households()) + levels = uk_draw_levels(inputs, definition, supports) + draw = uk_draw_frame(inputs, levels) + assigned = assign_atomic(draw, definition, supports) + for (_, row), level, case in zip( + assigned.iterrows(), levels, TOY_HOUSEHOLDS, strict=True + ): + support = supports[row["atomic_area_system"]] + position = int( + np.flatnonzero(support.arrays["area"] == row["atomic_area_code"])[0] + ) + assert support.arrays["frs_region"][position] == case.region + if level in {"L0", "L1"}: + assert support.arrays[IMD_DECILE_COLUMN][position] == str(case.imd_decile) + weight = ( + uk_band_weight_map(row["atomic_area_system"])[case.council_tax_band] + if level in {"L0", "L2"} + else DEFAULT_WEIGHT_COLUMN + ) + assert int(support.arrays[weight][position]) > 0 + + +def test_the_fallback_reasons_name_each_inputs_status(definition, supports) -> None: + inputs = uk_draw_inputs(_households()) + levels = uk_draw_levels(inputs, definition, supports) + reasons = uk_fallback_reasons(inputs, levels, definition) + assert reasons.index.equals(inputs.index) + for (_, row), case, level in zip( + reasons.iterrows(), TOY_HOUSEHOLDS, levels, strict=True + ): + assert row["decile"] in UK_FALLBACK_STATUSES["decile"] + assert row["band"] in UK_FALLBACK_STATUSES["band"] + if level == "L0": + assert (row["decile"], row["band"]) == ("kept", "kept") + if not case.imd_decile: + assert row["decile"] == "unobserved" + if case.region == "NORTHERN_IRELAND": + assert row["band"] == "none" + # London's band-H household in decile 1 keeps its decile and drops the band; + # its decile-9 household drops the decile and keeps the band. + assert reasons.iloc[1].tolist() == ["kept", "dropped"] + assert reasons.iloc[4].tolist() == ["dropped", "kept"] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_conditioned_area_supports.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_conditioned_area_supports.py index 7ef052b78f..5252897cb2 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_conditioned_area_supports.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_conditioned_area_supports.py @@ -30,6 +30,17 @@ ) from microcosm.build.spec_engine import load_bundle from microcosm.build.spec_engine.compiler_ir import compile_spec +from microcosm.build.uk_runtime.atomic_area_conditioning import ( + BAND_INPUT_COLUMN, + DECILE_INPUT_COLUMN, + IDENTITY_COLUMN, + REGION_COLUMN, + UK_DRAW_CONSTRAINTS, + uk_band_weight_map, + uk_conditioned_assignment_definition, + uk_draw_levels, + uk_system_fallback_levels, +) from microcosm.build.uk_runtime.atomic_area_support import ( BAND_WEIGHT_COLUMNS, BAND_WEIGHT_SCALE, @@ -474,3 +485,63 @@ def test_mounted_scottish_band_weights_follow_each_data_zone(provenance) -> None # households, so its weight is that share times households, rounded. share = frame["zone"].map(totals[column] / totals["households"]) assert np.all(np.abs(frame[column] - frame["households"] * share) <= 1.0) + + +def test_mounted_chooser_meets_exactly_the_committed_zero_mass_cells( + provenance, +) -> None: + """The conditioned draw's fallback ladder (microcosm#931) on the mounted + supports: every combination of region, decile and band, each missing too, + takes the first level whose cell the committed zero-mass table does not + list for the weight it samples by there.""" + + payloads = {system: _mounted(system) for system in SYSTEMS} + definition = uk_conditioned_assignment_definition(payloads, seed=0) + supports = {system: decode_atomic_support(payloads[system]) for system in SYSTEMS} + matched = {UK_DRAW_CONSTRAINTS[column]: column for column in UK_DRAW_CONSTRAINTS} + for system in SYSTEMS: + support = supports[system] + empty = { + (tuple(entry["constraints"]), entry["weight"]): { + tuple(cell) for cell in entry["cells"] + } + for entry in provenance["supports"][system]["zero_mass_cells"] + } + bands = uk_band_weight_map(system) + cases = [ + { + REGION_COLUMN: region, + DECILE_INPUT_COLUMN: decile, + BAND_INPUT_COLUMN: band, + } + for region in sorted(set(support.arrays["frs_region"].tolist())) + for decile in (*IMD_DECILES, pd.NA) + for band in (*bands, pd.NA) + ] + inputs = pd.DataFrame(cases, dtype=pd.StringDtype(storage="python")) + inputs.insert( + 0, + IDENTITY_COLUMN, + pd.array([f"combination-{n}" for n in range(len(cases))], dtype="string"), + ) + expected = [] + for case in cases: + for level in uk_system_fallback_levels(system): + if any(pd.isna(case[column]) for column in level.constraints): + continue + if level.weight_by and pd.isna(case[BAND_INPUT_COLUMN]): + continue + columns = tuple( + support_column + for support_column, column in matched.items() + if column in level.constraints + ) + weight = ( + bands[case[BAND_INPUT_COLUMN]] if level.weight_by else "households" + ) + cell = tuple(case[matched[column]] for column in columns) + if cell not in empty[(columns, weight)]: + expected.append(level.name) + break + levels = uk_draw_levels(inputs, definition, supports) + assert levels.tolist() == expected, system diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_stochastic.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_stochastic.py index 0a53373f61..ace195a59b 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_stochastic.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_stochastic.py @@ -1,6 +1,15 @@ """Tests split from packages/microcosm-build/tests/test_uk_frs_stochastic.py.""" # ruff: noqa: F403, F405 +from microcosm.build.stochastic_assignment import sample_categorical_from_counts +from microcosm.build.uk_runtime.frs_brma import ( + UK_BRMA_BENUNIT_SALT, + UK_BRMA_HOUSEHOLD_PICK_SALT, + _benunit_regions, + add_frs_brma, + draw_household_brmas, + load_brma_count_resource, +) from microcosm.build.uk_runtime.receipt_counts import own_source_households from test_support.microcosm_build.uk_frs_stochastic import * @@ -205,6 +214,191 @@ def test_brma_missing_cell_fails_closed() -> None: assign_brma_by_cell(benunit, count_resource={"cells": {"LONDON": {}}}, seed=0) +_BRMA_REGIONS = ("LONDON", "NORTHERN_IRELAND", "SCOTLAND") +_LHA_CATEGORIES = ("A", "B", "C", "D", "E") + + +def _brma_frame() -> object: + """Twelve invented households in three regions, with one to three + benefit units each and one person per benefit unit.""" + + households = np.arange(1, 13) + units = [ + (household, household * 100 + unit) + for household in households.tolist() + for unit in range(1, 2 + household % 3) + ] + benunit_ids = [unit for _, unit in units] + person = pd.DataFrame( + { + "person_id": [unit * 10 + 1 for unit in benunit_ids], + "person_benunit_id": benunit_ids, + "person_household_id": [household for household, _ in units], + "age": [40] * len(units), + "is_uc_claimant": [True] * len(units), + } + ) + household = pd.DataFrame( + { + "household_id": households, + "region": [_BRMA_REGIONS[h % 3] for h in households.tolist()], + } + ) + return uk_national_frame( + person=person, + benunit=pd.DataFrame({"benunit_id": benunit_ids}), + household=household, + household_weights=np.full(len(households), 10.0), + time_period="2024", + ) + + +def _brma_lha(benunit: pd.DataFrame) -> list[str]: + return [_LHA_CATEGORIES[int(unit) % 5] for unit in benunit["benunit_id"]] + + +def test_add_frs_brma_draws_what_the_cell_draw_and_household_pick_give() -> None: + """The stage's values are today's: each benefit unit's cell draw under + salt "brma", then each household's pick under "brma:household_pick", + both keyed on ids with seed 0; the benefit-unit table is left as it was.""" + + frame = _brma_frame() + resource = load_brma_count_resource() + person, benunit = frame.table("person"), frame.table("benunit") + household = frame.table("household") + assert (UK_BRMA_BENUNIT_SALT, UK_BRMA_HOUSEHOLD_PICK_SALT) == ( + "brma", + "brma:household_pick", + ) + cells = benunit.assign( + LHA_category=_brma_lha(benunit), + region=_benunit_regions(person, household, benunit), + ) + drawn = assign_brma_by_cell(cells, count_resource=resource, seed=0) + expected = collapse_benunit_brma_to_household( + person, cells[["benunit_id"]].assign(brma=drawn), household, seed=0 + ) + result = add_frs_brma( + frame, lha_category=_brma_lha(benunit), count_resource=resource + ) + assert result.table("household")["brma"].tolist() == expected.tolist() + assert result.table("benunit").equals(benunit) + # The cell draw, written out for one benefit unit. + first = cells.iloc[0] + uniform = stable_identity_uniforms([first["benunit_id"]], seed=0, salt="brma") + counts = resource["cells"][first["region"]][first["LHA_category"]] + assert drawn[0] == sample_categorical_from_counts(uniform, counts=counts)[0] + + +def test_add_frs_brma_picks_each_households_brma_as_written_out() -> None: + """The household pick, which sets frs_brma's live values, written out + without the module's pick: the uniform keyed on the household's id under + salt "brma:household_pick" and seed 0 picks among its benefit units' + cell draws in ascending benefit-unit id.""" + + frame = _brma_frame() + resource = load_brma_count_resource() + person, benunit = frame.table("person"), frame.table("benunit") + household = frame.table("household") + lha = _brma_lha(benunit) + cells = benunit.assign( + LHA_category=lha, region=_benunit_regions(person, household, benunit) + ) + drawn = pd.Series( + assign_brma_by_cell(cells, count_resource=resource, seed=0), + index=benunit["benunit_id"].to_numpy(), + ) + owner = person.drop_duplicates("person_benunit_id").set_index("person_benunit_id")[ + "person_household_id" + ] + result = add_frs_brma(frame, lha_category=lha, count_resource=resource) + picked = result.table("household")["brma"].tolist() + choices = 0 + for row, household_id in enumerate(household["household_id"].tolist()): + units = sorted(int(unit) for unit in owner.index[owner == household_id]) + options = [str(drawn[unit]) for unit in units] + (uniform,) = stable_identity_uniforms( + [household_id], seed=0, salt="brma:household_pick" + ) + expected = options[min(int(uniform * len(options)), len(options) - 1)] + assert picked[row] == expected + choices += len(set(options)) > 1 + # The invented frame holds households whose benefit units drew BRMAs + # that differ, so the pick is exercised. + assert choices + + +def test_brma_draw_of_a_household_subset_takes_the_whole_frames_draws() -> None: + """Every uniform is keyed on its own id alone, so a subset of the + households, given every benefit unit or only its own, takes the whole + frame's BRMAs; benefit units outside the subset are never read.""" + + frame = _brma_frame() + resource = load_brma_count_resource() + person, benunit = frame.table("person"), frame.table("benunit") + household = frame.table("household") + lha = _brma_lha(benunit) + whole = add_frs_brma(frame, lha_category=lha, count_resource=resource) + whole_brma = whole.table("household")["brma"].to_numpy() + for region in _BRMA_REGIONS: + rows = (household["region"] == region).to_numpy() + subset = household.loc[rows] + drawn = draw_household_brmas( + person, benunit, subset, lha_category=lha, count_resource=resource + ) + assert drawn.tolist() == whole_brma[rows].tolist() + members = person.loc[person["person_household_id"].isin(subset["household_id"])] + own = benunit["benunit_id"].isin(members["person_benunit_id"]).to_numpy() + alone = draw_household_brmas( + members, + benunit.loc[own], + subset, + lha_category=[value for value, keep in zip(lha, own, strict=True) if keep], + count_resource=resource, + ) + assert alone.tolist() == whole_brma[rows].tolist() + # A category no cell holds, on the other households' benefit units. + foreign = [value if keep else "Z" for value, keep in zip(lha, own, strict=True)] + assert ( + draw_household_brmas( + person, benunit, subset, lha_category=foreign, count_resource=resource + ).tolist() + == whole_brma[rows].tolist() + ) + + +def test_brma_refusals_name_no_household_or_benefit_unit_id() -> None: + frame = _brma_frame() + resource = load_brma_count_resource() + person, benunit = frame.table("person"), frame.table("benunit") + household = frame.table("household") + lha = _brma_lha(benunit) + # A benefit unit whose members sit in two households. + split = person.copy() + split.loc[split.index[0], "person_household_id"] = int( + household["household_id"].iloc[-1] + ) + split = pd.concat([split, person.iloc[[0]].assign(person_id=99_999)]) + with pytest.raises(ValueError, match="splits a benefit unit") as refused: + draw_household_brmas( + split, benunit, household, lha_category=lha, count_resource=resource + ) + assert not any(character.isdigit() for character in str(refused.value)) + # A household without a benefit unit drawn for it. + with pytest.raises(ValueError, match="has no benefit unit") as refused: + collapse_benunit_brma_to_household( + person, + pd.DataFrame({"benunit_id": benunit["benunit_id"].iloc[:1], "brma": ["X"]}), + household, + seed=0, + ) + assert not any(character.isdigit() for character in str(refused.value)) + with pytest.raises(ValueError, match="must align"): + draw_household_brmas( + person, benunit, household, lha_category=lha[:-1], count_resource=resource + ) + + def test_uc_take_up_population_excludes_units_without_a_working_age_adult() -> None: """A unit with no UC claimant or partner of working age is never drawn.""" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rfdr_area_layer.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rfdr_area_layer.py index 3656a24d74..4007ea1d4c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rfdr_area_layer.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rfdr_area_layer.py @@ -138,8 +138,11 @@ def test_households_take_their_area_flag_and_northern_ireland_none() -> None: "uk_ni_data_zone_2021": 1, "flagged": 2, } - with pytest.raises(ValueError, match="no support system"): + with pytest.raises(ValueError, match="holds another value") as refused: area_flags(layer=layer, area_system=["legacy"], area_code=["X"]) + # The accepted systems are named; the frame's own value is not. + assert "uk_ni_data_zone_2021" in str(refused.value) + assert "legacy" not in str(refused.value) @pytest.mark.parametrize( @@ -246,6 +249,71 @@ def test_a_sampled_rung_may_flag_none_but_still_refuses_malformed_codes() -> Non ) +_EW = "uk_ew_output_area_2021" +_SCOTLAND = "uk_scotland_output_area_2022" +_ROSTER = {_EW: ["E00000001", "E00000002"], _SCOTLAND: ["S00135400", "S00135401"]} + + +def test_a_roster_holds_the_layer_to_its_supports_areas() -> None: + """microcosm#931, C-6: the clone-first spine holds the layer to its + supports' areas, so a layer of another code vintage refuses whatever the + households drew, and a system none of whose households is flagged does + not.""" + + layer = load_rfdr_area_layer(_layer()) + households = { + "area_system": [_EW, _SCOTLAND, "uk_ni_data_zone_2021"], + "area_code": ["E00000001", "S00135401", "N20000001"], + } + flags, counts = area_flags( + layer=layer, **households, require_flagged=False, roster=_ROSTER + ) + assert flags.tolist() == [False, False, False] + assert counts["flagged"] == 0 + # The roster's codes in any iterable, NI's absent: the check reads the + # Great Britain systems alone. + flags, _ = area_flags( + layer=layer, + area_system=[_EW], + area_code=["E00000002"], + require_flagged=False, + roster={system: tuple(codes) for system, codes in _ROSTER.items()}, + ) + assert flags.tolist() == [True] + off_roster = {**_ROSTER, _SCOTLAND: ["S00135401"]} + with pytest.raises(ValueError, match="support does not hold") as refused: + area_flags(layer=layer, **households, require_flagged=False, roster=off_roster) + # It names the system, never the layer's code or a household's. + assert "S00135400" not in str(refused.value) + assert "S00135401" not in str(refused.value) + with pytest.raises(ValueError, match="roster of support areas"): + area_flags( + layer=layer, **households, require_flagged=False, roster={_EW: _ROSTER[_EW]} + ) + with pytest.raises(ValueError, match="roster of support areas"): + area_flags( + layer=layer, **households, require_flagged=False, roster=[("x", "y")] + ) + # Without a roster the flags are what they were. + unrostered, _ = area_flags(layer=layer, **households, require_flagged=False) + assert unrostered.tolist() == [False, False, False] + + +def test_the_malformed_code_refusal_names_no_code_and_counts_nothing() -> None: + layer = load_rfdr_area_layer(_layer()) + with pytest.raises(ValueError, match="well-formed") as refused: + area_flags( + layer=layer, + area_system=[_EW] * 3, + area_code=["E00000002", "e00000002", "E0000002"], + roster=_ROSTER, + require_flagged=False, + ) + text = str(refused.value) + assert "e00000002" not in text and "E0000002" not in text + assert not any(character.isdigit() for character in text.replace(_EW, "")) + + def test_the_committed_layer_was_built_from_the_registered_nspl_zip() -> None: """Anthony's review of microcosm#1420, 7b: the committed layer comes from the NSPL zip Chronicle registers, not from a reduction of it, and its diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_geography.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_geography.py new file mode 100644 index 0000000000..14479761a3 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_geography.py @@ -0,0 +1,1867 @@ +"""The clone-first spine's area stages (microcosm#931). + +G2 draws each geographic clone's area, G3 resolves its local authority, G4 +its BRMA and G5 its fuel-relief flag, and the SPI-copy area stage re-keys and +re-places the rows the SPI support channel copied. No stage declares them +yet, so each transform here runs as a stage built from the reviewed +declaration spine_geography states. The tests run them on toy conditioned +supports whose codes resolve against the committed resources (test_support's +uk_conditioned_support_fixtures), on invented households cloned and keyed by +the spine's own expansion and identity steps. G4 and the copy stage read +``LHA_category`` from a recording stand-in for the rules engine, which gives +each benefit unit an invented category from its id and keeps every frame it +is handed; the copy stage runs on a stand-in for spi_support_channel and +spi_income_band_donors that copies invented households at the support +channel's two ordinals. + +They hold: each household in its region at the level its inputs give it; the +written dtypes; clone 0's areas at K = 1, 2 and 3; a draw that does not +depend on row order; the validation, which passes on the prepared frame, +fails on a fallback row's raw decile and fails the stage where the written +ladder disagrees with the mapping; Great Britain's BRMAs off the layer and +Northern Ireland's as frs_brma draws them; every copy's own key, area, local +authority, BRMA (Northern Ireland's drawn on the copy's own benefit units, +after one engine call over those copies alone, which the UK adapter's clone +rule takes as one simulation at every K) and flag, with every other row left +as it was; every refusal, count-free and naming no key or code of a +household, the frame's refusals before any engine call; and every evidence +count byte-equal at K = 1 and K = 3. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import replace +from importlib.resources import files + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.atomic_geography import ( + assign_atomic, + derive_geography, + validate_geography, +) +from microcosm.build.source_manifest import ( + SourceOperationSpec, + SourceStageSpec, + _reject_executable_parameter_keys, + _reject_incumbent_dependencies, +) +from microcosm.build.uk_runtime import copy_id_table, spine_geography +from microcosm.build.uk_runtime.atomic_area_conditioning import ( + DECILE_INPUT_COLUMN, + IDENTITY_COLUMN, + UK_SPINE_AREA_COLUMNS, + UK_SPINE_ATOMIC_AREA_SEED, + UK_SPINE_LADDER_COLUMNS, + uk_conditioned_assignment_definition, + uk_draw_frame, + uk_draw_inputs, + uk_draw_levels, +) +from microcosm.build.uk_runtime.atomic_area_support import IMD_DECILE_COLUMN, _profile +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.brma_area_layer import ( + GRAINS as BRMA_GRAINS, +) +from microcosm.build.uk_runtime.brma_area_layer import ( + load_brma_area_layer, +) +from microcosm.build.uk_runtime.frs_brma import ( + UK_BRMA_BENUNIT_SALT, + UK_BRMA_HOUSEHOLD_PICK_SALT, + add_frs_brma, + assign_brma_by_cell, + collapse_benunit_brma_to_household, + load_brma_count_resource, +) +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column +from microcosm.build.uk_runtime.household_lineage import uk_household_clone_paths +from microcosm.build.uk_runtime.local_authority_input import ( + local_authority_engine_key_by_code, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_national_frame, + uk_time_period, +) +from microcosm.build.uk_runtime.receipt_counts import rounded_count +from microcosm.build.uk_runtime.rfdr_area_layer import ( + RFDR_COLUMN, + load_rfdr_area_layer, +) +from microcosm.build.uk_runtime.spi_band_donors import ( + SPI_INCOME_BAND_DONOR_CLONE_INDEX, +) +from microcosm.build.uk_runtime.spine_geography import ( + _COPY_KINDS, + _NORTHERN_IRELAND_REGION, + AREA_BRMA_STAGE_NAME, + ATOMIC_AREA_ASSIGN_STAGE_NAME, + BRMA_COUNT_ARTIFACT_ROLE, + BRMA_LAYER_ARTIFACT_ROLE, + BRMA_LAYER_RESOURCE, + LOCAL_AUTHORITY_ARTIFACT_ROLE, + LOCAL_AUTHORITY_STAGE_NAME, + RFDR_ARTIFACT_ROLE, + RFDR_LAYER_RESOURCE, + RFDR_STAGE_NAME, + SPI_COPY_AREAS_STAGE_NAME, + UK_AREA_BRMA_OUTPUT_COLUMNS, + UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS, + UK_LOCAL_AUTHORITY_OUTPUT_COLUMNS, + UK_RFDR_OUTPUT_COLUMNS, + UK_SPI_COPY_AREAS_REWRITE_COLUMNS, + UKAreaBRMAStageTransform, + UKAtomicAreaAssignStageTransform, + UKLocalAuthorityStageTransform, + UKRFDRStageTransform, + UKSPICopyAreasStageTransform, + area_brma_operation_parameters, + assign_uk_spi_copy_areas, + atomic_area_assign_operation_parameters, + draw_uk_spine_areas, + flag_uk_spine_rfdr, + load_committed_rfdr_layer, + local_authority_operation_parameters, + rfdr_operation_parameters, + spi_copy_areas_operation_parameters, + uk_spi_copy_keys, +) +from microcosm.frame import Frame, MassChangeRecord +from microcosm.frame.adapters.policyengine_uk import PolicyEngineUKEngine +from test_support.microcosm_build.uk_atomic_support_fixtures import toy_support_payloads +from test_support.microcosm_build.uk_conditioned_support_fixtures import ( + EW, + NI, + SCOT, + TOY_AREAS, + TOY_FLAGGED_AREAS, + TOY_HOUSEHOLDS, + ToyHousehold, + committed_rfdr_areas, + toy_conditioned_support_payloads, + toy_conditioned_supports, + toy_spine_frame, +) + +_STRING = pd.StringDtype(storage="python") +_CLONE = ladder_clone_index_column("household") +_SYSTEM_OF_REGION = {"NORTHERN_IRELAND": NI, "SCOTLAND": SCOT} +#: The toy households the committed RFDR layer flags: drawn into the Hawes +#: and the Highlands-and-Islands toy areas, the only areas with mass in their +#: cells. +_FLAGGED = [ + position + for position, case in enumerate(TOY_HOUSEHOLDS) + if (case.region, case.imd_decile) in {("YORKSHIRE", 6), ("SCOTLAND", 4)} +] + + +def _stage(name: str, **overrides: object) -> SourceStageSpec: + """A stage built from the reviewed declaration spine_geography states, + with ``overrides`` replacing fields.""" + + brma_artifacts = ( + (BRMA_LAYER_ARTIFACT_ROLE, BRMA_LAYER_RESOURCE), + (BRMA_COUNT_ARTIFACT_ROLE, "brma_private_rented_households.json"), + ) + reviewed = { + ATOMIC_AREA_ASSIGN_STAGE_NAME: ( + atomic_area_assign_operation_parameters(), + UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS, + (), + (), + ), + LOCAL_AUTHORITY_STAGE_NAME: ( + local_authority_operation_parameters(), + UK_LOCAL_AUTHORITY_OUTPUT_COLUMNS, + ((LOCAL_AUTHORITY_ARTIFACT_ROLE, "local_authority_names.json"),), + (), + ), + AREA_BRMA_STAGE_NAME: ( + area_brma_operation_parameters(), + UK_AREA_BRMA_OUTPUT_COLUMNS, + brma_artifacts, + (), + ), + RFDR_STAGE_NAME: ( + rfdr_operation_parameters(), + UK_RFDR_OUTPUT_COLUMNS, + ((RFDR_ARTIFACT_ROLE, RFDR_LAYER_RESOURCE),), + (), + ), + SPI_COPY_AREAS_STAGE_NAME: ( + spi_copy_areas_operation_parameters(), + (), + ( + ("frs_release", "frs_release.json"), + (LOCAL_AUTHORITY_ARTIFACT_ROLE, "local_authority_names.json"), + *brma_artifacts, + (RFDR_ARTIFACT_ROLE, RFDR_LAYER_RESOURCE), + ), + UK_SPI_COPY_AREAS_REWRITE_COLUMNS, + ), + } + operations, outputs, artifacts, rewrites = reviewed[name] + stage = SourceStageSpec( + stage=name, + survey="Family Resources Survey 2024-25", + source="Microcosm structural operator (microcosm#931)", + grain="household", + artifacts=tuple( + { + "role": role, + "kind": "public_aggregate_reference", + "resource": resource, + "format": "json", + "runtime_sha256_required": True, + } + for role, resource in artifacts + ), + operations=tuple( + SourceOperationSpec(kind=kind, parameters=dict(parameters)) + for kind, parameters in operations + ), + outputs=tuple(outputs), + rewrites=tuple(rewrites), + ) + return replace(stage, **overrides) + + +@pytest.fixture(scope="module") +def payloads() -> dict[str, bytes]: + return toy_conditioned_support_payloads() + + +@pytest.fixture(scope="module") +def supports() -> dict: + return toy_conditioned_supports() + + +@pytest.fixture(scope="module") +def definition(payloads) -> dict: + return uk_conditioned_assignment_definition( + payloads, seed=UK_SPINE_ATOMIC_AREA_SEED + ) + + +def _g2(payloads) -> UKAtomicAreaAssignStageTransform: + return UKAtomicAreaAssignStageTransform( + stage=_stage(ATOMIC_AREA_ASSIGN_STAGE_NAME), supports=payloads + ) + + +def _g3() -> UKLocalAuthorityStageTransform: + return UKLocalAuthorityStageTransform(stage=_stage(LOCAL_AUTHORITY_STAGE_NAME)) + + +def _g5(payloads) -> UKRFDRStageTransform: + return UKRFDRStageTransform(stage=_stage(RFDR_STAGE_NAME), supports=payloads) + + +def _run(payloads, clones: int, households=TOY_HOUSEHOLDS) -> dict[str, object]: + """The three stages in order on ``clones`` clones of ``households``.""" + + g2, g3, g5 = _g2(payloads), _g3(), _g5(payloads) + frames = {"keyed": toy_spine_frame(clones=clones, households=households)} + frames["g2"] = g2(frames["keyed"]) + frames["g3"] = g3(frames["g2"]) + frames["g5"] = g5(frames["g3"]) + return { + "frames": frames, + "evidence": { + stage: transform.checkpoint_metadata()["evidence"] + for stage, transform in (("g2", g2), ("g3", g3), ("g5", g5)) + }, + } + + +@pytest.fixture(scope="module") +def runs(payloads) -> dict[int, dict[str, object]]: + return {clones: _run(payloads, clones) for clones in (1, 2, 3)} + + +def _household(runs, clones: int, stage: str = "g5") -> pd.DataFrame: + return runs[clones]["frames"][stage].table("household") + + +def _clone_zero(household: pd.DataFrame) -> pd.DataFrame: + return household.loc[household[_CLONE] == 0].reset_index(drop=True) + + +# --- The toy supports --------------------------------------------------------------- + + +def _committed(name: str) -> dict: + return json.loads(files("microcosm.build.uk").joinpath(name).read_text("utf-8")) + + +def test_the_toy_supports_resolve_against_the_committed_resources() -> None: + """Every code G3, G4, G5, G7 and G8 read resolves in its committed resource.""" + + roster = local_authority_engine_key_by_code() + layer = _committed("brma_area_layer.json")["brmas"] + rents = set(_committed("brma_rent_quantiles.json")["brmas"]) + knots = set( + _committed("high_value_home_value_knots.json")["levels"]["constituency"] + ) + flagged = committed_rfdr_areas() + brma_of = { + grain: { + code: (region, brma) + for region, by_brma in regions.items() + for brma, codes in by_brma.items() + for code in codes + } + for grain, regions in layer.items() + } + for area in TOY_AREAS: + assert area.local_authority_code in roster + if area.system == NI: + continue + key, grain = ( + (area.lsoa_code, "ew_lsoa21") + if area.system == EW + else (area.oa_code, "scotland_oa22") + ) + region, brma = brma_of[grain][key] + assert region == area.region + assert brma in rents + if area.system == EW: + assert area.constituency_code in knots + assert (area.oa_code in flagged[area.system]) is ( + area.oa_code == TOY_FLAGGED_AREAS[area.system] + ) + # Every system is exercised, Northern Ireland's districts on the roster. + assert {area.system for area in TOY_AREAS} == {EW, SCOT, NI} + + +def test_each_great_britain_support_holds_every_flagged_output_area(supports) -> None: + for system, codes in committed_rfdr_areas().items(): + areas = set(supports[system].arrays["area"].tolist()) + assert set(codes) <= areas + # The fillers carry no households, so the draw never meets them. + filler = np.isin( + supports[system].arrays["area"], + [code for code in codes if code != TOY_FLAGGED_AREAS[system]], + ) + assert not np.asarray(supports[system].arrays["households"])[filler].any() + small = toy_conditioned_supports(rfdr_roster=False) + assert len(small[SCOT].arrays["area"]) == sum(a.system == SCOT for a in TOY_AREAS) + + +# --- The reviewed declarations ------------------------------------------------------ + + +def test_the_reviewed_operations_are_declarative() -> None: + for operations in ( + atomic_area_assign_operation_parameters(), + local_authority_operation_parameters(), + rfdr_operation_parameters(), + ): + for kind, parameters in operations: + _reject_executable_parameter_keys(parameters, context=kind) + _reject_incumbent_dependencies(parameters, context=kind) + assert json.loads(json.dumps(parameters)) == parameters + ((_, g2),) = atomic_area_assign_operation_parameters() + assert g2["seed"] == UK_SPINE_ATOMIC_AREA_SEED == 0 + assert g2["layers"] == list(UK_SPINE_LADDER_COLUMNS) + assert [level["name"] for level in g2["fallback_levels"]] == [ + "L0", + "L1", + "L2", + "L3", + ] + ((_, g5),) = rfdr_operation_parameters() + assert g5["require_flagged"] is False + + +@pytest.mark.parametrize( + ("name", "build"), + [ + (ATOMIC_AREA_ASSIGN_STAGE_NAME, lambda stage, p: _g2_with(stage, p)), + ( + LOCAL_AUTHORITY_STAGE_NAME, + lambda stage, p: UKLocalAuthorityStageTransform(stage=stage), + ), + ( + RFDR_STAGE_NAME, + lambda stage, p: UKRFDRStageTransform(stage=stage, supports=p), + ), + ], +) +def test_each_transform_runs_only_as_its_reviewed_stage(name, build, payloads) -> None: + stage = _stage(name) + build(stage, payloads) + operation = stage.operations[0] + drifted = SourceOperationSpec( + kind=operation.kind, parameters={**operation.parameters, "extra": 1} + ) + for overrides, message in ( + ({"stage": "frs_brma"}, "Expected stage"), + ({"grain": "benunit"}, "grain must be 'household'"), + ({"outputs": stage.outputs[:-1]}, "writes exactly its own columns"), + ({"rewrites": stage.outputs[:1]}, "writes exactly its own columns"), + ({"operations": (drifted,)}, "drifted"), + ({"operations": ()}, "operation order drifted"), + ( + { + "artifacts": ( + *stage.artifacts, + { + "role": "other", + "resource": "x.json", + "runtime_sha256_required": True, + }, + ) + }, + "and no other", + ), + ): + with pytest.raises(ValueError, match=message): + build(_stage(name, **overrides), payloads) + with pytest.raises(TypeError, match="SourceStageSpec"): + build(object(), payloads) + + +def _g2_with(stage, payloads): + return UKAtomicAreaAssignStageTransform(stage=stage, supports=payloads) + + +def test_the_artifact_must_require_its_runtime_digest(payloads) -> None: + stage = _stage(RFDR_STAGE_NAME) + (artifact,) = stage.artifacts + loose = replace(stage, artifacts=({**artifact, "runtime_sha256_required": False},)) + with pytest.raises(ValueError, match="runtime_sha256_required"): + UKRFDRStageTransform(stage=loose, supports=payloads) + + +def test_checkpoint_metadata_needs_a_completed_run(payloads) -> None: + for transform in (_g2(payloads), _g3(), _g5(payloads)): + with pytest.raises(RuntimeError, match="completed stage run"): + transform.checkpoint_metadata() + + +# --- G2 ----------------------------------------------------------------------------- + + +def test_g2_writes_the_assignment_and_ladder_columns_as_python_strings( + runs, supports +) -> None: + household = _household(runs, 3, "g2") + assert UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS == UK_SPINE_AREA_COLUMNS + for column in UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS: + assert household[column].dtype == _STRING, column + assert household[column].notna().all() + assert set(household["atomic_area_basis"]) == {"assigned"} + # Every ladder column is the support's own row for the drawn area. + for _, row in household.iterrows(): + support = supports[row["atomic_area_system"]] + position = int( + np.flatnonzero(support.arrays["area"] == row["atomic_area_code"])[0] + ) + assert row["oa_code"] == row["atomic_area_code"] + for column in UK_SPINE_LADDER_COLUMNS[1:]: + assert row[column] == support.arrays[column][position], column + + +def test_g2_keeps_every_row_id_weight_and_earlier_column(runs) -> None: + keyed = runs[3]["frames"]["keyed"] + placed = runs[3]["frames"]["g2"] + for entity in keyed.entities: + before, after = keyed.table(entity), placed.table(entity) + assert after[before.columns].equals(before) + assert np.array_equal( + keyed.weights_for("household").values, placed.weights_for("household").values + ) + assert placed.mass_log == keyed.mass_log + assert list(placed.table("household").columns) == [ + *keyed.table("household").columns, + *UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS, + ] + + +def test_every_household_stays_in_its_region_at_its_level(runs, supports) -> None: + household = _household(runs, 3, "g2") + levels = np.tile([case.level for case in TOY_HOUSEHOLDS], 3) + cases = TOY_HOUSEHOLDS * 3 + for (_, row), case, level in zip(household.iterrows(), cases, levels, strict=True): + system = _SYSTEM_OF_REGION.get(case.region, EW) + assert row["atomic_area_system"] == system + support = supports[system] + position = int( + np.flatnonzero(support.arrays["area"] == row["atomic_area_code"])[0] + ) + assert support.arrays["frs_region"][position] == case.region + if level in {"L0", "L1"}: + assert support.arrays[IMD_DECILE_COLUMN][position] == str(case.imd_decile) + # London's band-H household in decile 1 falls to L1: its area is a + # decile-1 area, none of which holds band H. + band_h = household.iloc[1] + support = supports[EW] + position = int( + np.flatnonzero(support.arrays["area"] == band_h["atomic_area_code"])[0] + ) + assert support.arrays[IMD_DECILE_COLUMN][position] == "1" + assert int(support.arrays["households_band_h"][position]) == 0 + + +def test_g2_evidence_counts_sources_by_system_level_and_reason(runs) -> None: + evidence = runs[1]["evidence"]["g2"] + by_level: dict[str, int] = {} + for case in TOY_HOUSEHOLDS: + by_level[case.level] = by_level.get(case.level, 0) + 1 + assert evidence["source_households"] == rounded_count(len(TOY_HOUSEHOLDS)) + assert evidence["source_households_by_level"] == { + level: rounded_count(by_level.get(level, 0)) + for level in ("L0", "L1", "L2", "L3") + } + systems = [_SYSTEM_OF_REGION.get(case.region, EW) for case in TOY_HOUSEHOLDS] + assert evidence["source_households_by_system"] == { + system: rounded_count(systems.count(system)) for system in (EW, SCOT, NI) + } + assert set(evidence["source_households_by_fallback_reason"]) == {"decile", "band"} + assert evidence["validation"] == { + "outcome": "pass", + "scope": "atomic_geography_mapping_integrity", + } + assert evidence["seed"] == 0 + assert evidence["stream"][1] == "uk-spine-atomic-area-v1" + # No exact count: the validation's row count is replaced by sources. + assert "households" not in evidence["validation"] + + +def test_clone_zero_draws_at_k_1_2_and_3_what_k_1_draws(runs) -> None: + columns = [IDENTITY_COLUMN, *UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS] + single = _clone_zero(_household(runs, 1))[columns] + for clones in (2, 3): + assert _clone_zero(_household(runs, clones))[columns].equals(single) + # Each clone draws its own key, so the areas vary across clones. + household = _household(runs, 3) + assert household[IDENTITY_COLUMN].is_unique + assert ( + household.groupby("source_household_id")["atomic_area_code"].nunique() > 1 + ).any() + + +def test_the_draw_does_not_depend_on_row_order(runs, definition, supports) -> None: + household = runs[3]["frames"]["keyed"].table("household") + shuffled = household.sample(frac=1.0, random_state=931) + ordered = draw_uk_spine_areas(household, definition=definition, supports=supports) + permuted = draw_uk_spine_areas(shuffled, definition=definition, supports=supports) + assert permuted.columns.index.equals(shuffled.index) + assert permuted.columns.loc[household.index].equals(ordered.columns) + assert np.array_equal( + pd.Series(permuted.levels, index=shuffled.index) + .loc[household.index] + .to_numpy(), + ordered.levels, + ) + + +def test_validation_passes_on_the_prepared_frame_and_fails_on_a_raw_decile( + runs, definition, supports +) -> None: + household = runs[1]["frames"]["keyed"].table("household") + inputs = uk_draw_inputs(household) + levels = uk_draw_levels(inputs, definition, supports) + prepared = uk_draw_frame(inputs, levels) + placed = pd.concat( + [prepared, assign_atomic(prepared, definition, supports)], axis=1 + ) + frame = pd.concat([placed, derive_geography(placed, definition, supports)], axis=1) + assert validate_geography(frame, definition, supports)["outcome"] == "pass" + # London's decile-9 household falls to L2: no area holds its decile, so + # its area violates the raw decile the prepared frame dropped. + (fallback,) = [ + position + for position, case in enumerate(TOY_HOUSEHOLDS) + if (case.region, case.imd_decile) == ("LONDON", 9) + ] + assert levels[fallback] == "L2" + raw = frame.copy() + raw.iloc[fallback, raw.columns.get_loc(DECILE_INPUT_COLUMN)] = inputs[ + DECILE_INPUT_COLUMN + ].iloc[fallback] + with pytest.raises(ValueError, match="violates observed geography") as refused: + validate_geography(raw, definition, supports) + assert household[IDENTITY_COLUMN].iloc[fallback] not in str(refused.value) + + +def _corrupt_first_msoa(monkeypatch) -> None: + """Make the area draw write, for its first row, another row's MSOA: the + shared validation, which re-derives the ladder itself, must refuse.""" + + real = spine_geography.derive_geography + + def corrupt(households, spec, supports): + derived = real(households, spec, supports) + values = derived["msoa_code"].tolist() + other = next(value for value in values if value != values[0]) + derived.iloc[0, derived.columns.get_loc("msoa_code")] = other + return derived + + monkeypatch.setattr(spine_geography, "derive_geography", corrupt) + + +def test_g2_and_the_copy_stage_refuse_a_ladder_the_mapping_does_not_give( + monkeypatch, payloads, copy_runs +) -> None: + keyed = toy_spine_frame(clones=1) + copied = copy_runs[1]["frames"]["copied"] + g2 = _g2(payloads) + engine = _RecordingEngine() + copy_stage = _copy_stage(payloads, engine) + _corrupt_first_msoa(monkeypatch) + with pytest.raises(ValueError, match="derived geography differs") as refused: + g2(keyed) + assert g2.last_result is None + assert not any(c.isdigit() for c in str(refused.value)) + with pytest.raises(ValueError, match="derived geography differs"): + copy_stage(copied) + assert copy_stage.last_result is None + assert engine.calls == [] + + +def test_g2_refuses_a_frame_that_already_carries_an_output(runs, payloads) -> None: + placed = runs[1]["frames"]["g2"] + with pytest.raises(ValueError, match="already carries") as refused: + _g2(payloads)(placed) + assert "atomic_area_code" in str(refused.value) + + +def test_g2_refuses_a_frame_without_the_geography_key(runs, payloads) -> None: + keyed = runs[1]["frames"]["keyed"] + tables = {entity: keyed.table(entity) for entity in keyed.entities} + tables["household"] = tables["household"].drop(columns=[IDENTITY_COLUMN]) + bare = Frame( + tables, + keyed.schema, + {"household": keyed.weights_for("household")}, + keyed.strata, + mass_log=keyed.mass_log, + metadata=keyed.metadata, + ) + with pytest.raises(ValueError, match="lacks") as refused: + _g2(payloads)(bare) + assert IDENTITY_COLUMN in str(refused.value) + + +def test_g2_refuses_supports_that_are_not_conditioned() -> None: + with pytest.raises(ValueError, match="conditioned support columns differ"): + _g2(toy_support_payloads()) + + +# --- G3 ----------------------------------------------------------------------------- + + +def test_g3_resolves_each_drawn_code_to_its_engine_name(runs) -> None: + household = _household(runs, 3, "g3") + roster = local_authority_engine_key_by_code() + assert household["local_authority"].dtype == _STRING + assert household["local_authority"].tolist() == [ + roster[code] for code in household["local_authority_code"] + ] + assert list(household.columns)[-1] == "local_authority" + + +def test_g3_refuses_an_off_roster_code_naming_no_code(runs) -> None: + placed = runs[1]["frames"]["g2"] + tables = {entity: placed.table(entity) for entity in placed.entities} + household = tables["household"].copy() + invented = "E06999999" + assert invented not in local_authority_engine_key_by_code() + household["local_authority_code"] = household["local_authority_code"].mask( + household.index == household.index[0], invented + ) + tables["household"] = household + frame = Frame( + tables, + placed.schema, + {"household": placed.weights_for("household")}, + placed.strata, + mass_log=placed.mass_log, + metadata=placed.metadata, + ) + with pytest.raises(ValueError, match="off the April 2023") as refused: + _g3()(frame) + assert invented not in str(refused.value) + + +def test_g3_refuses_a_frame_without_the_code_or_already_resolved(runs) -> None: + with pytest.raises(ValueError, match="lacks"): + _g3()(runs[1]["frames"]["keyed"]) + with pytest.raises(ValueError, match="already carries"): + _g3()(runs[1]["frames"]["g3"]) + + +# --- G5 ----------------------------------------------------------------------------- + + +def test_g5_flags_the_households_drawn_into_flagged_areas(runs) -> None: + flagged_areas = committed_rfdr_areas() + for clones in (1, 3): + household = _household(runs, clones) + assert household[RFDR_COLUMN].dtype == np.dtype(bool) + systems = household["atomic_area_system"].to_numpy(dtype=object) + codes = household["atomic_area_code"].to_numpy(dtype=object) + expected = [ + system in flagged_areas and code in flagged_areas[system] + for system, code in zip(systems, codes, strict=True) + ] + assert household[RFDR_COLUMN].tolist() == expected + # The households whose cells hold one area, a flagged one, are flagged + # in every clone; Northern Ireland never is. + for clone in range(clones): + for position in _FLAGGED: + assert household[RFDR_COLUMN].iloc[ + position + len(TOY_HOUSEHOLDS) * clone + ] + assert not household.loc[systems == NI, RFDR_COLUMN].any() + + +def test_g5_refuses_a_layer_that_flags_an_area_off_its_support_roster(runs) -> None: + small = toy_conditioned_supports(rfdr_roster=False) + layer = load_rfdr_area_layer( + files("microcosm.build.uk").joinpath(RFDR_LAYER_RESOURCE).read_bytes() + ) + placed = runs[1]["frames"]["g3"] + with pytest.raises(ValueError, match="does not hold") as refused: + flag_uk_spine_rfdr(placed, layer=layer, layer_sha256="0" * 64, supports=small) + text = str(refused.value) + assert not any( + code in text for codes in committed_rfdr_areas().values() for code in codes + ) + with pytest.raises(ValueError, match="does not hold"): + UKRFDRStageTransform( + stage=_stage(RFDR_STAGE_NAME), + supports=toy_conditioned_support_payloads(rfdr_roster=False), + )(placed) + + +def test_g5_reports_a_system_with_no_flagged_household_without_refusing( + payloads, +) -> None: + unflagged = [ + case + for case in TOY_HOUSEHOLDS + if (case.region, case.imd_decile) != ("YORKSHIRE", 6) + ] + run = _run(payloads, 1, unflagged) + household = run["frames"]["g5"].table("household") + england_wales = household["atomic_area_system"] == EW + assert england_wales.any() and not household.loc[england_wales, RFDR_COLUMN].any() + evidence = run["evidence"]["g5"] + assert evidence["no_flagged_household"] == {EW: True, SCOT: False} + assert evidence["flagged_source_households"][EW] == 0 + assert evidence["require_flagged"] is False + + +def test_g5_still_refuses_a_malformed_drawn_code(runs, payloads) -> None: + placed = runs[1]["frames"]["g3"] + tables = {entity: placed.table(entity) for entity in placed.entities} + household = tables["household"].copy() + first = household.index[household["atomic_area_system"] == EW][0] + malformed = household.loc[first, "atomic_area_code"].lower() + household.loc[first, "atomic_area_code"] = malformed + tables["household"] = household + frame = Frame( + tables, + placed.schema, + {"household": placed.weights_for("household")}, + placed.strata, + mass_log=placed.mass_log, + metadata=placed.metadata, + ) + with pytest.raises(ValueError, match="well-formed") as refused: + _g5(payloads)(frame) + assert malformed not in str(refused.value) + + +def test_g5_evidence_counts_flagged_sources_among_clone_zero(runs) -> None: + single = _clone_zero(_household(runs, 1)) + expected = { + system: rounded_count( + single.loc[ + single[RFDR_COLUMN] & (single["atomic_area_system"] == system), + "source_household_id", + ].nunique() + ) + for system in (EW, SCOT) + } + for clones in (1, 3): + evidence = runs[clones]["evidence"]["g5"] + assert evidence["counted_among"] == "clone_0" + assert evidence["flagged_source_households"] == expected + assert evidence["no_flagged_household"] == {EW: False, SCOT: False} + assert evidence["roster"] == "pass" + assert evidence["require_flagged"] is False + assert evidence["method"] == "uk-rfdr-area-layer-v1" + + +def test_g5_counts_flagged_sources_in_clone_zero_where_later_clones_differ( + payloads, +) -> None: + """Region-level draws land in a flagged area in some clones and not in + others, so a union over clones grows with K; the evidence counts clone 0.""" + + cases = [ToyHousehold("YORKSHIRE", 0, "D", False, "L3")] * 12 + [ + ToyHousehold("SCOTLAND", 0, "D", False, "L3") + ] * 12 + g2, g3, g5 = _g2(payloads), _g3(), _g5(payloads) + g5(g3(g2(toy_spine_frame(clones=3, households=cases)))) + household = g5.last_result.frame.table("household") + flagged = household[RFDR_COLUMN].to_numpy() + zero = (household[_CLONE] == 0).to_numpy() + systems = household["atomic_area_system"].to_numpy(dtype=object) + sources = household["source_household_id"].to_numpy() + + def distinct(rows) -> int: + return len(set(sources[rows].tolist())) + + in_clone_zero = {s: distinct(zero & flagged & (systems == s)) for s in (EW, SCOT)} + in_every_clone = {s: distinct(flagged & (systems == s)) for s in (EW, SCOT)} + # The invented frame tells the two counts apart. + assert in_clone_zero != in_every_clone + assert dict(g5.last_result.flagged_by_system) == in_clone_zero + assert dict(g5.last_result.no_flagged_household) == { + system: count == 0 for system, count in in_clone_zero.items() + } + + +# --- G4 and the SPI-copy area stage: stand-ins -------------------------------------- + +#: The LHA categories the recording engine hands out: the count table's five. +_LHA_CATEGORIES = ("A", "B", "C", "D", "E") +_SUPPORT_CHANNEL = "household_support_channel" +_SUPPORT_ORDINAL = "household_support_clone_index" + + +class _RecordingEngine: + """A stand-in for the UK rules engine that keeps every call. + + Each benefit unit's ``LHA_category`` is invented from its own id, modulo + five; a copy's ids add whole multiples of the copy-id table's radices, so + a copy takes its parent's category, as a whole-row copy takes it from + the engine. ``misaligned`` answers one benefit unit short. + """ + + def __init__(self, *, misaligned: bool = False, country: str = "uk") -> None: + self.country = country + self.misaligned = misaligned + self.calls: list[tuple[Frame, tuple[str, ...], str]] = [] + + def materialize(self, frame, variables, period): + self.calls.append((frame, tuple(variables), str(period))) + ids = frame.table("benunit")["benunit_id"].to_numpy(dtype=np.int64) + values = np.asarray([_LHA_CATEGORIES[int(i) % 5] for i in ids], dtype=object) + return {"LHA_category": values[:-1] if self.misaligned else values} + + +def _lha(frame: Frame) -> np.ndarray: + return _RecordingEngine().materialize( + frame, ("LHA_category",), uk_time_period(frame) + )["LHA_category"] + + +def _g4(engine: object) -> UKAreaBRMAStageTransform: + return UKAreaBRMAStageTransform(stage=_stage(AREA_BRMA_STAGE_NAME), engine=engine) + + +def _copy_stage(payloads, engine: object) -> UKSPICopyAreasStageTransform: + return UKSPICopyAreasStageTransform( + stage=_stage(SPI_COPY_AREAS_STAGE_NAME), engine=engine, supports=payloads + ) + + +def _replace_household(frame: Frame, household: pd.DataFrame) -> Frame: + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = household + return Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _copy_picks(source: int, clone: int) -> tuple[int, ...]: + """The invented copies spi_support_channel and spi_income_band_donors + make of a household, by ordinal. + + In clone 0 every household is copied (1) and every third one is a band + donor too (2), by its source alone, so clone 0's copies are the same at + every clone count; a later clone copies other households, so a union of + copied sources over clones grows with K. + """ + + if clone == 0: + return (1, 2) if source % 3 == 0 else (1,) + return tuple( + ordinal + for ordinal, picked in ( + (1, (source + clone) % 2 == 0), + (2, (source + clone) % 3 == 0), + ) + if picked + ) + + +def _with_support_channel(frame: Frame, picks=_copy_picks) -> Frame: + """``frame`` as spi_support_channel and spi_income_band_donors leave it, + for these tests alone. + + Every row is on the support channel's FRS side at ordinal 0; each + household ``picks`` names is copied whole at the channel's ordinals, + every id minted in the copy-id table's ``spi_support_channel`` field, each + copy at half its parent's invented weight, with a mass record for the + copies. No other lineage column is added. + """ + + person, benunit = frame.table("person"), frame.table("benunit") + household = frame.table("household").copy() + household[_SUPPORT_CHANNEL] = "frs" + household[_SUPPORT_ORDINAL] = np.zeros(len(household), dtype=np.int64) + weights = frame.weights_for("household").values + persons, benunits, households = [person], [benunit], [household] + masses = [weights] + for ordinal in (1, 2): + positions = [ + position + for position, (source, clone) in enumerate( + zip( + household["source_household_id"].tolist(), + household[_CLONE].tolist(), + strict=True, + ) + ) + if ordinal in picks(int(source), int(clone)) + ] + if not positions: + continue + copied = household.iloc[positions].copy() + members = person.loc[person["person_household_id"].isin(copied["household_id"])] + members = members.copy() + units = benunit.loc[benunit["benunit_id"].isin(members["person_benunit_id"])] + units = units.copy() + for column in ("person_id", "person_household_id", "person_benunit_id"): + members[column] = copy_id_table.mint( + members[column], "spi_support_channel", ordinal + ) + units["benunit_id"] = copy_id_table.mint( + units["benunit_id"], "spi_support_channel", ordinal + ) + copied["household_id"] = copy_id_table.mint( + copied["household_id"], "spi_support_channel", ordinal + ) + copied[_SUPPORT_CHANNEL] = "spi" + copied[_SUPPORT_ORDINAL] = np.int64(ordinal) + persons.append(members) + benunits.append(units) + households.append(copied) + masses.append(weights[positions] * 0.5) + values = np.concatenate(masses) + record = MassChangeRecord( + entity="household", + old_total=float(weights.sum()), + new_total=float(values.sum()), + declared_factor=None, + reason="invented copies standing in for the SPI support channel", + ) + return uk_national_frame( + person=pd.concat(persons, ignore_index=True), + benunit=pd.concat(benunits, ignore_index=True), + household=pd.concat(households, ignore_index=True), + time_period=uk_time_period(frame), + household_weights=values, + mass_log=(*frame.mass_log, record), + ) + + +def _copy_run(payloads, base: Frame, picks=_copy_picks) -> dict[str, object]: + """G4 on ``base`` (G5's frame), the stand-in for the support channel's + two stages, then the copy stage, each with an engine of its own.""" + + engines = {"g4": _RecordingEngine(), "copy": _RecordingEngine()} + g4, copy_stage = _g4(engines["g4"]), _copy_stage(payloads, engines["copy"]) + frames = {"g4": g4(base)} + frames["copied"] = _with_support_channel(frames["g4"], picks) + frames["copy"] = copy_stage(frames["copied"]) + return { + "frames": frames, + "engines": engines, + "results": {"g4": g4.last_result, "copy": copy_stage.last_result}, + "evidence": { + "g4": g4.checkpoint_metadata()["evidence"], + "copy": copy_stage.checkpoint_metadata()["evidence"], + }, + } + + +@pytest.fixture(scope="module") +def copy_runs(payloads, runs) -> dict[int, dict[str, object]]: + return { + clones: _copy_run(payloads, runs[clones]["frames"]["g5"]) + for clones in (1, 2, 3) + } + + +def _copies(household: pd.DataFrame) -> np.ndarray: + return (household[_SUPPORT_CHANNEL] == "spi").to_numpy(dtype=bool) + + +def _committed_brma_layer(): + return load_brma_area_layer( + files("microcosm.build.uk").joinpath(BRMA_LAYER_RESOURCE).read_bytes() + ) + + +# --- G4 and the copy stage: their reviewed declarations ----------------------------- + + +def test_the_g4_and_copy_stage_operations_are_declarative_and_agree() -> None: + for operations in ( + area_brma_operation_parameters(), + spi_copy_areas_operation_parameters(), + ): + for kind, parameters in operations: + _reject_executable_parameter_keys(parameters, context=kind) + _reject_incumbent_dependencies(parameters, context=kind) + assert json.loads(json.dumps(parameters)) == parameters + predictors = ( + "materialize_rules_engine_predictors", + {"predictors": ["LHA_category"], "consumed_only": True}, + ) + (g4_predictors, (_, g4)) = area_brma_operation_parameters() + (copy_predictors, (_, copy)) = spi_copy_areas_operation_parameters() + assert g4_predictors == copy_predictors == predictors + # frs_brma's seed and salts, which the Northern Ireland draws keep. + assert (g4["output"], g4["seed"]) == ("brma", 0) + assert ( + g4["salts"] + == { + "benunit": UK_BRMA_BENUNIT_SALT, + "household_pick": UK_BRMA_HOUSEHOLD_PICK_SALT, + } + == {"benunit": "brma", "household_pick": "brma:household_pick"} + ) + # The copy stage names G2's definition with G2's stream and seed, and + # G4's count-table draw. + ((_, g2),) = atomic_area_assign_operation_parameters() + assert copy["area_draw"] == { + "definition_of": ATOMIC_AREA_ASSIGN_STAGE_NAME, + "level": g2["fallback_levels"][-1]["name"], + "stream": g2["stream"], + "seed": g2["seed"], + } + assert copy["area_draw"]["level"] == "L3" + shared = set(copy["brma"]) & set(g4) + assert shared >= {"seed", "salts", "count_resource", "cell_columns"} + assert {key: copy["brma"][key] for key in shared} == { + key: g4[key] for key in shared + } + # The engine reads Northern Ireland's copies alone, in one simulation over + # every clone's copies (C-9). + assert (copy["brma"]["engine_households"], copy["brma"]["engine_simulations"]) == ( + "northern_ireland_copies", + "one", + ) + assert copy["rewrites"] == list(UK_SPI_COPY_AREAS_REWRITE_COLUMNS) + + +def test_the_copy_kinds_and_northern_irelands_region_match_their_owners() -> None: + """The ordinals and the region the stages spell for themselves.""" + + bound = copy_id_table.ordinal_bound("spi_support_channel") + assert set(_COPY_KINDS) == set(range(1, bound)) + assert _COPY_KINDS[SPI_INCOME_BAND_DONOR_CLONE_INDEX] == "income_band_donor" + assert _profile(NI)[2] == (_NORTHERN_IRELAND_REGION,) + + +@pytest.mark.parametrize("name", [AREA_BRMA_STAGE_NAME, SPI_COPY_AREAS_STAGE_NAME]) +def test_g4_and_the_copy_stage_run_only_as_their_reviewed_stages( + name, payloads +) -> None: + def build(stage): + if name == AREA_BRMA_STAGE_NAME: + return UKAreaBRMAStageTransform(stage=stage, engine=_RecordingEngine()) + return UKSPICopyAreasStageTransform( + stage=stage, engine=_RecordingEngine(), supports=payloads + ) + + stage = _stage(name) + build(stage) + *head, last = stage.operations + drifted = SourceOperationSpec( + kind=last.kind, parameters={**last.parameters, "extra": 1} + ) + loose = {**stage.artifacts[-1], "runtime_sha256_required": False} + for overrides, message in ( + ({"stage": "frs_brma"}, "Expected stage"), + ({"grain": "benunit"}, "grain must be 'household'"), + ({"outputs": ("brma_extra",)}, "writes exactly its own columns"), + ({"rewrites": ("brma_extra",)}, "writes exactly its own columns"), + ({"operations": (*head, drifted)}, "drifted"), + ({"operations": tuple(head)}, "operation order drifted"), + ({"artifacts": stage.artifacts[:-1]}, "and no other"), + ({"artifacts": (*stage.artifacts[:-1], loose)}, "runtime_sha256_required"), + ): + with pytest.raises(ValueError, match=message): + build(_stage(name, **overrides)) + with pytest.raises(TypeError, match="SourceStageSpec"): + build(object()) + + +def test_g4_and_the_copy_stage_need_a_run_and_the_uk_engine(payloads, runs) -> None: + for transform in ( + _g4(_RecordingEngine()), + _copy_stage(payloads, _RecordingEngine()), + ): + with pytest.raises(RuntimeError, match="completed stage run"): + transform.checkpoint_metadata() + with pytest.raises(ValueError, match="does not match"): + _g4(_RecordingEngine(country="us"))(runs[1]["frames"]["g5"]) + + +# --- G4 ----------------------------------------------------------------------------- + + +def test_g4_reads_great_britain_brmas_off_the_committed_layer(copy_runs) -> None: + layer = _committed_brma_layer() + for clones in (1, 3): + household = copy_runs[clones]["frames"]["g4"].table("household") + assert household["brma"].dtype == np.dtype(object) + for grain, spec in BRMA_GRAINS.items(): + rows = (household["atomic_area_system"] == spec.system).to_numpy() + assert rows.any() + assert household.loc[rows, "brma"].tolist() == [ + layer.brma_by_area[grain][code] + for code in household.loc[rows, spec.key] + ] + + +def test_g4_draws_northern_ireland_as_frs_brma_does(runs, copy_runs) -> None: + """Northern Ireland's benefit units draw with frs_brma's keys, so each + of its households takes the BRMA frs_brma gives it, at every K.""" + + for clones in (1, 3): + base = runs[clones]["frames"]["g5"] + reference = add_frs_brma( + base, lha_category=_lha(base), count_resource=load_brma_count_resource() + ).table("household") + household = copy_runs[clones]["frames"]["g4"].table("household") + northern_ireland = (household["atomic_area_system"] == NI).to_numpy() + assert northern_ireland.sum() == 4 * clones + assert ( + household["brma"].to_numpy()[northern_ireland].tolist() + == reference["brma"].to_numpy()[northern_ireland].tolist() + ) + + +def test_g4_makes_one_engine_call_over_the_whole_frame(runs, copy_runs) -> None: + for clones in (1, 3): + (call,) = copy_runs[clones]["engines"]["g4"].calls + frame, variables, period = call + base = runs[clones]["frames"]["g5"] + assert variables == ("LHA_category",) + assert period == uk_time_period(base) + assert frame is base + + +def test_g4_keeps_every_row_and_appends_brma(runs, copy_runs) -> None: + base, placed = runs[3]["frames"]["g5"], copy_runs[3]["frames"]["g4"] + for entity in base.entities: + before, after = base.table(entity), placed.table(entity) + assert after[before.columns].equals(before) + assert list(placed.table("household").columns) == [ + *base.table("household").columns, + "brma", + ] + assert np.array_equal( + base.weights_for("household").values, placed.weights_for("household").values + ) + assert placed.mass_log == base.mass_log + + +def test_g4_clone_zero_takes_at_k_3_the_brmas_of_k_1(copy_runs) -> None: + single, triple = ( + _clone_zero(copy_runs[clones]["frames"]["g4"].table("household"))["brma"] + for clones in (1, 3) + ) + assert triple.tolist() == single.tolist() + + +def _g4_refusal(frame: Frame, engine: object | None = None) -> str: + with pytest.raises(ValueError) as refused: + _g4(engine if engine is not None else _RecordingEngine())(frame) + return str(refused.value) + + +def test_g4_refuses_an_area_off_the_layer_naming_no_code(runs) -> None: + base = runs[1]["frames"]["g5"] + household = base.table("household").copy() + first = household.index[household["atomic_area_system"] == EW][0] + invented = "E01999999" + assert invented not in _committed_brma_layer().brma_by_area["ew_lsoa21"] + household.loc[first, "lsoa_code"] = invented + engine = _RecordingEngine() + text = _g4_refusal(_replace_household(base, household), engine) + assert "not on the layer" in text + assert invented not in text + # Refused before the engine call. + assert engine.calls == [] + + +def test_g4_refuses_an_area_the_layer_places_in_another_region(runs) -> None: + base = runs[1]["frames"]["g5"] + household = base.table("household").copy() + first = household.index[household["region"] == "LONDON"][0] + household.loc[first, "region"] = "SOUTH_EAST" + engine = _RecordingEngine() + text = _g4_refusal(_replace_household(base, household), engine) + assert "another FRS region" in text + assert household.loc[first, "lsoa_code"] not in text + # Refused by G4's own check, before the engine call. + assert engine.calls == [] + + +def test_g4_refuses_a_system_or_region_it_cannot_place(runs) -> None: + base = runs[1]["frames"]["g5"] + household = base.table("household") + unknown = household.copy() + unknown["atomic_area_system"] = unknown["atomic_area_system"].mask( + unknown.index == unknown.index[0], "uk_invented_system" + ) + engine = _RecordingEngine() + text = _g4_refusal(_replace_household(base, unknown), engine) + assert "holds another value" in text and "uk_invented_system" not in text + moved = household.copy() + first = moved.index[moved["atomic_area_system"] == NI][0] + moved.loc[first, "region"] = "WALES" + assert "disagree" in _g4_refusal(_replace_household(base, moved), engine) + assert engine.calls == [] + + +def test_g4_refuses_misaligned_engine_values_and_a_written_or_missing_column( + runs, copy_runs +) -> None: + base = runs[1]["frames"]["g5"] + assert "not aligned" in _g4_refusal(base, _RecordingEngine(misaligned=True)) + # A frame G4 cannot place is refused before the engine call. + engine = _RecordingEngine() + assert "already carries" in _g4_refusal(copy_runs[1]["frames"]["g4"], engine) + household = base.table("household").drop(columns=["lsoa_code"]) + text = _g4_refusal(_replace_household(base, household), engine) + assert "lacks" in text and "lsoa_code" in text + assert engine.calls == [] + + +def test_g4_evidence_counts_sources_by_grain_and_northern_ireland_drawn( + copy_runs, +) -> None: + systems = [_SYSTEM_OF_REGION.get(case.region, EW) for case in TOY_HOUSEHOLDS] + digest = hashlib.sha256( + files("microcosm.build.uk").joinpath(BRMA_LAYER_RESOURCE).read_bytes() + ).hexdigest() + for clones in (1, 3): + evidence = copy_runs[clones]["evidence"]["g4"] + assert evidence["source_households_by_grain"] == { + "ew_lsoa21": rounded_count(systems.count(EW)), + "scotland_oa22": rounded_count(systems.count(SCOT)), + } + # Sources, not benefit units. + assert evidence["northern_ireland_source_households_drawn"] == rounded_count( + systems.count(NI) + ) + assert evidence["layer_sha256"] == digest + assert evidence["method"] == "uk-brma-area-layer-v1" + assert (evidence["seed"], evidence["output"]) == (0, "brma") + + +# --- The SPI-copy area stage -------------------------------------------------------- + + +def test_the_copy_stage_runs_on_the_support_channel_lineage_alone(copy_runs) -> None: + """The frame carries the support channel's lineage and the clone index + alone; the full-path reader, which needs the CGT steps, refuses it.""" + + copied = copy_runs[3]["frames"]["copied"].table("household") + assert not {"household_is_cgt_support_copy", "cgt_support_copy_index"} & set( + copied.columns + ) + with pytest.raises(ValueError, match="lacks lineage column"): + uk_household_clone_paths(copied) + assert copy_runs[3]["frames"]["copy"].table("household")[IDENTITY_COLUMN].is_unique + + +def _expected_copy_keys(household: pd.DataFrame) -> list[str]: + """Each copy's key over its source, vintage and copy path, written out.""" + + copies = household.loc[_copies(household)] + return [ + household_draw_key( + source="frs", + source_vintage="2024_25", + source_household_id=int(source), + clone_path=(("spi_support_channel", int(ordinal)),) + + ((("geographic_support", int(clone)),) if clone else ()), + ) + for source, ordinal, clone in zip( + copies["source_household_id"], + copies[_SUPPORT_ORDINAL], + copies[_CLONE], + strict=True, + ) + ] + + +def test_every_copy_takes_a_key_of_its_own_over_its_copy_path(copy_runs) -> None: + for clones in (1, 3): + before = copy_runs[clones]["frames"]["copied"].table("household") + after = copy_runs[clones]["frames"]["copy"].table("household") + copies = _copies(before) + assert copies.any() + # Each copy came in with its parent's key, and leaves with its own. + assert not before[IDENTITY_COLUMN].is_unique + assert after[IDENTITY_COLUMN].is_unique + assert after.loc[copies, IDENTITY_COLUMN].tolist() == _expected_copy_keys( + before + ) + assert uk_spi_copy_keys(before)[1] == _expected_copy_keys(before) + assert after.loc[~copies, IDENTITY_COLUMN].equals( + before.loc[~copies, IDENTITY_COLUMN] + ) + + +def test_a_copys_key_is_the_full_path_readers_once_later_steps_exist( + copy_runs, +) -> None: + """With the CGT steps' columns at ordinal 0, the full-path reader gives + every household, base row or copy, the key it carries.""" + + household = copy_runs[3]["frames"]["copy"].table("household").copy() + zeros = np.zeros(len(household), dtype=np.int64) + for column in ("cgt_support_copy_index", "cgt_residential_clone_index"): + household[column] = zeros + for column in ( + "household_is_cgt_support_copy", + "household_is_capital_gains_clone", + "household_is_cgt_residential_clone", + ): + household[column] = zeros.astype(bool) + sources, paths = uk_household_clone_paths(household) + assert [ + household_draw_key( + source="frs", + source_vintage="2024_25", + source_household_id=source, + clone_path=path, + ) + for source, path in zip(sources, paths, strict=True) + ] == household[IDENTITY_COLUMN].tolist() + + +def test_clone_zero_copies_keep_their_keys_and_places_as_k_grows(copy_runs) -> None: + columns = [IDENTITY_COLUMN, *UK_SPINE_AREA_COLUMNS, "local_authority", "brma"] + + def clone_zero_copies(clones: int) -> pd.DataFrame: + household = copy_runs[clones]["frames"]["copy"].table("household") + rows = household.loc[_copies(household) & (household[_CLONE] == 0).to_numpy()] + return ( + rows.set_index(["source_household_id", _SUPPORT_ORDINAL])[ + [*columns, RFDR_COLUMN] + ] + .sort_index() + .astype({column: object for column in columns}) + ) + + for clones in (2, 3): + assert clone_zero_copies(clones).equals(clone_zero_copies(1)), clones + + +def test_the_copy_stage_leaves_every_other_row_and_table_as_it_was(copy_runs) -> None: + for clones in (1, 3): + before_frame = copy_runs[clones]["frames"]["copied"] + after_frame = copy_runs[clones]["frames"]["copy"] + before = before_frame.table("household") + after = after_frame.table("household") + copies = _copies(before) + assert list(after.columns) == list(before.columns) + assert after.loc[~copies].equals(before.loc[~copies]) + kept = [c for c in before.columns if c not in UK_SPI_COPY_AREAS_REWRITE_COLUMNS] + assert after[kept].equals(before[kept]) + for entity in ("person", "benunit"): + assert after_frame.table(entity).equals(before_frame.table(entity)) + assert np.array_equal( + after_frame.weights_for("household").values, + before_frame.weights_for("household").values, + ) + assert after_frame.mass_log == before_frame.mass_log + assert after_frame.metadata == before_frame.metadata + + +def test_each_rewrite_keeps_its_columns_dtype(copy_runs) -> None: + before = copy_runs[3]["frames"]["copied"].table("household") + after = copy_runs[3]["frames"]["copy"].table("household") + assert after.dtypes.equals(before.dtypes) + for column in (IDENTITY_COLUMN, *UK_SPINE_AREA_COLUMNS, "local_authority"): + assert after[column].dtype == _STRING, column + assert after["brma"].dtype == np.dtype(object) + assert after[RFDR_COLUMN].dtype == np.dtype(bool) + + +def test_every_copy_draws_its_area_at_l3_in_its_region( + copy_runs, definition, supports +) -> None: + for clones in (1, 3): + before = copy_runs[clones]["frames"]["copied"].table("household") + after = copy_runs[clones]["frames"]["copy"].table("household") + copies = _copies(after) + rows = after.loc[copies] + levels = uk_draw_levels(uk_draw_inputs(rows), definition, supports) + assert set(levels.tolist()) == {"L3"} + for _, row in rows.iterrows(): + support = supports[row["atomic_area_system"]] + position = int( + np.flatnonzero(support.arrays["area"] == row["atomic_area_code"])[0] + ) + assert support.arrays["frs_region"][position] == row["region"] + assert row["oa_code"] == row["atomic_area_code"] + for column in UK_SPINE_LADDER_COLUMNS[1:]: + assert row[column] == support.arrays[column][position], column + # Each copy's area is the draw of its own key, not its parent's area. + drawn = draw_uk_spine_areas(rows, definition=definition, supports=supports) + assert rows[list(UK_SPINE_AREA_COLUMNS)].equals(drawn.columns) + assert ( + after.loc[copies, "atomic_area_code"].to_numpy() + != before.loc[copies, "atomic_area_code"].to_numpy() + ).any() + + +def test_every_copy_takes_its_areas_local_authority_brma_and_flag(copy_runs) -> None: + roster = local_authority_engine_key_by_code() + layer = _committed_brma_layer() + flagged = committed_rfdr_areas() + for clones in (1, 3): + household = copy_runs[clones]["frames"]["copy"].table("household") + rows = household.loc[_copies(household)] + assert rows["local_authority"].tolist() == [ + roster[code] for code in rows["local_authority_code"] + ] + for grain, spec in BRMA_GRAINS.items(): + great_britain = rows.loc[rows["atomic_area_system"] == spec.system] + assert len(great_britain) + assert great_britain["brma"].tolist() == [ + layer.brma_by_area[grain][code] for code in great_britain[spec.key] + ] + assert rows[RFDR_COLUMN].tolist() == [ + system in flagged and code in flagged[system] + for system, code in zip( + rows["atomic_area_system"], rows["atomic_area_code"], strict=True + ) + ] + + +def test_northern_ireland_copies_draw_their_own_brma_on_their_own_benefit_units( + copy_runs, +) -> None: + """C-9, María's ruling of 2026-11-11: one engine call over the Northern + Ireland copies' households alone, then frs_brma's draw keyed on the + copies' own benefit units and households, with its seed and salts.""" + + differs = [] + for clones in (1, 3): + run = copy_runs[clones] + frame = run["frames"]["copy"] + household = frame.table("household") + person, benunit = frame.table("person"), frame.table("benunit") + northern_ireland = _copies(household) & ( + household["atomic_area_system"] == NI + ).to_numpy(dtype=bool) + ids = household.loc[northern_ireland, "household_id"] + assert len(ids) + (call,) = run["engines"]["copy"].calls + seen, variables, period = call + assert (variables, period) == (("LHA_category",), uk_time_period(frame)) + assert seen.table("household")["household_id"].tolist() == ids.tolist() + # One simulation over every clone's copies: no clone index to batch by. + assert _CLONE not in seen.table("household").columns + members = person.loc[person["person_household_id"].isin(ids)] + assert sorted(seen.table("person")["person_id"]) == sorted(members["person_id"]) + units = benunit.loc[benunit["benunit_id"].isin(members["person_benunit_id"])] + assert ( + seen.table("benunit")["benunit_id"].tolist() == units["benunit_id"].tolist() + ) + cells = units[["benunit_id"]].assign( + region="NORTHERN_IRELAND", + LHA_category=[_LHA_CATEGORIES[int(i) % 5] for i in units["benunit_id"]], + ) + drawn = assign_brma_by_cell( + cells, count_resource=load_brma_count_resource(), seed=0 + ) + expected = collapse_benunit_brma_to_household( + person, + cells[["benunit_id"]].assign(brma=drawn), + household.loc[northern_ireland], + seed=0, + ) + assert household.loc[northern_ireland, "brma"].tolist() == expected.tolist() + # The copies arrived with their parents' BRMAs. + before = run["frames"]["copied"].table("household") + differs += ( + before.loc[northern_ireland, "brma"].to_numpy() != expected + ).tolist() + # Drawn, not inherited: some copy's BRMA is not its parent's. + assert any(differs) + + +def test_the_copy_stage_calls_no_engine_without_northern_ireland_copies( + payloads, runs +) -> None: + base = runs[1]["frames"]["g5"] + household = base.table("household") + northern_ireland = set( + household.loc[household["region"] == "NORTHERN_IRELAND", "source_household_id"] + ) + run = _copy_run( + payloads, + base, + picks=lambda source, clone: () if source in northern_ireland else (1,), + ) + assert run["engines"]["copy"].calls == [] + evidence = run["evidence"]["copy"] + assert evidence["copy_source_households"]["spi_support_copy"][NI] == 0 + assert run["frames"]["copy"].table("household")[IDENTITY_COLUMN].is_unique + + +class _GuardedEngine(_RecordingEngine): + """The recording engine, refusing as the UK adapter built with + ``clones=K`` refuses: its own batching decision, which builds no engine, + runs on every frame it is handed and records the simulations it would + run.""" + + def __init__(self, clones: int) -> None: + super().__init__() + self.adapter = PolicyEngineUKEngine(clones=clones) + self.simulations: list[int] = [] + + def materialize(self, frame, variables, period): + self.simulations.append(self.adapter.simulation_batches(frame)) + return super().materialize(frame, variables, period) + + +@pytest.mark.parametrize("clones", [2, 3]) +def test_the_copy_stages_engine_call_is_one_simulation_at_every_clone_count( + payloads, runs, clones +) -> None: + """Stage 16 draws its SPI share per clone, so a later clone may hold no + Northern Ireland copy. The adapter refuses a call over part of its pool, + so the call over the copies alone carries no clone index and runs as one + simulation over every clone's copies, as at one clone (C-9).""" + + base = runs[clones]["frames"]["g5"] + household = base.table("household") + northern_ireland = set( + household.loc[household["region"] == "NORTHERN_IRELAND", "source_household_id"] + ) + + def picks(source: int, clone: int) -> tuple[int, ...]: + if clone > 0 and source in northern_ireland: + return () + return _copy_picks(source, clone) + + g4_engine = _GuardedEngine(clones) + copied = _with_support_channel(_g4(g4_engine)(base), picks) + assert g4_engine.simulations == [clones] + copies = copied.table("household") + in_clone_zero = ( + _copies(copies) + & (copies["atomic_area_system"] == NI).to_numpy() + & (copies[_CLONE] == 0).to_numpy() + ) + assert in_clone_zero.any() + # Kept, the clone index would put the copies of clone 0 alone before an + # engine built for ``clones`` clones, which refuses them. + subset = copied.select( + copied.table("person")["person_household_id"] + .isin(copies.loc[in_clone_zero, "household_id"]) + .to_numpy(dtype=bool) + ) + with pytest.raises(ValueError, match="over a sub-pool"): + PolicyEngineUKEngine(clones=clones).simulation_batches(subset) + engine = _GuardedEngine(clones) + copy_stage = _copy_stage(payloads, engine) + placed = copy_stage(copied).table("household") + assert engine.simulations == [1] + (call,) = engine.calls + assert call[0].table("household")["household_id"].tolist() == ( + copies.loc[in_clone_zero, "household_id"].tolist() + ) + assert placed.loc[in_clone_zero, "brma"].notna().all() + assert copy_stage.last_result.evidence()["brma"]["engine_simulations"] == "one" + + +def test_the_copy_stage_says_its_checks_did_not_run_without_a_copy( + payloads, runs +) -> None: + run = _copy_run(payloads, runs[1]["frames"]["g5"], picks=lambda source, clone: ()) + evidence = run["evidence"]["copy"] + assert evidence["validation"] == { + "outcome": "not_run", + "scope": "atomic_geography_mapping_integrity", + } + assert evidence["rfdr"]["roster"] == "not_run" + assert run["engines"]["copy"].calls == [] + assert ( + run["frames"]["copy"] + .table("household") + .equals(run["frames"]["copied"].table("household")) + ) + assert all( + count == 0 + for counts in evidence["copy_source_households"].values() + for count in counts.values() + ) + + +def _layer_without_toy_areas(*, misplace: bool) -> object: + """The committed BRMA layer with every England-and-Wales toy LSOA dropped, + or, with ``misplace``, placed in another FRS region than its own.""" + + layer = _committed_brma_layer() + lsoas = {area.lsoa_code for area in TOY_AREAS if area.system == EW} + brmas = {grain: dict(codes) for grain, codes in layer.brma_by_area.items()} + regions = {grain: dict(codes) for grain, codes in layer.region_by_area.items()} + for code in lsoas: + if misplace: + regions["ew_lsoa21"][code] = ( + "SOUTH_EAST" if regions["ew_lsoa21"][code] != "SOUTH_EAST" else "LONDON" + ) + else: + del brmas["ew_lsoa21"][code] + del regions["ew_lsoa21"][code] + return replace(layer, brma_by_area=brmas, region_by_area=regions) + + +@pytest.mark.parametrize( + ("misplace", "message"), + [(False, "not on the layer"), (True, "another FRS region")], +) +def test_the_copy_stage_checks_the_copies_areas_before_its_engine_call( + payloads, definition, supports, copy_runs, misplace, message +) -> None: + copied = copy_runs[1]["frames"]["copied"] + household = copied.table("household") + # Northern Ireland copies are present, so an engine call would follow. + assert ( + _copies(household) & (household["atomic_area_system"] == NI).to_numpy() + ).any() + rfdr_layer, rfdr_sha256 = load_committed_rfdr_layer() + engine = _RecordingEngine() + with pytest.raises(ValueError, match=message) as refused: + assign_uk_spi_copy_areas( + copied, + definition=definition, + supports=supports, + engine=engine, + brma_layer=_layer_without_toy_areas(misplace=misplace), + brma_layer_sha256="0" * 64, + count_resource=load_brma_count_resource(), + count_resource_sha256="0" * 64, + rfdr_layer=rfdr_layer, + rfdr_layer_sha256=rfdr_sha256, + ) + assert engine.calls == [] + text = str(refused.value) + assert not any(area.lsoa_code in text for area in TOY_AREAS) + + +def _copy_refusal(payloads, frame: Frame, engine: object | None = None) -> str: + with pytest.raises(ValueError) as refused: + _copy_stage(payloads, engine if engine is not None else _RecordingEngine())( + frame + ) + return str(refused.value) + + +def test_the_copy_stage_refuses_lineage_it_cannot_read(payloads, copy_runs) -> None: + # No support channel: the stage runs after spi_income_band_donors. + text = _copy_refusal(payloads, copy_runs[1]["frames"]["g4"]) + assert "'spi_support_channel' step" in text + copied = copy_runs[1]["frames"]["copied"] + household = copied.table("household") + # A later copy step has run. + later = household.copy() + marker = np.zeros(len(later), dtype=bool) + marker[0] = True + later["household_is_cgt_support_copy"] = marker + later["cgt_support_copy_index"] = marker.astype(np.int64) + text = _copy_refusal(payloads, _replace_household(copied, later)) + assert "before any later copy step" in text + # An ordinal the support channel never mints. + other = household.copy() + donor = other.index[other[_SUPPORT_ORDINAL] == 2][0] + other.loc[donor, _SUPPORT_ORDINAL] = 3 + assert "another ordinal" in _copy_refusal( + payloads, _replace_household(copied, other) + ) + + +def test_the_copy_stage_refuses_copies_whose_keys_repeat(payloads, runs) -> None: + """Two copies of one source on one path would share a key.""" + + placed = _g4(_RecordingEngine())(runs[1]["frames"]["g5"]) + household = placed.table("household").copy() + sources = household["source_household_id"].to_numpy().copy() + sources[1] = sources[0] + household["source_household_id"] = sources + copied = _with_support_channel(_replace_household(placed, household)) + text = _copy_refusal(payloads, copied) + assert "keys repeat" in text + keys = set(copied.table("household")[IDENTITY_COLUMN]) + assert not any(key in text for key in keys) + # No id or count: the issue reference is its only number. + assert not any(c.isdigit() for c in text.replace("microcosm#931", "")) + + +def test_the_copy_stage_refuses_a_missing_column_or_misaligned_engine_values( + payloads, copy_runs +) -> None: + copied = copy_runs[1]["frames"]["copied"] + without = copied.table("household").drop(columns=["brma"]) + text = _copy_refusal(payloads, _replace_household(copied, without)) + assert "lacks" in text and "brma" in text + text = _copy_refusal(payloads, copied, _RecordingEngine(misaligned=True)) + assert "not aligned" in text + + +def test_the_copy_stage_counts_copies_donors_and_flags_among_clone_zero( + copy_runs, +) -> None: + single = copy_runs[1]["frames"]["copy"].table("household") + rows = single.loc[_copies(single)] + + def sources(mask) -> int: + return rows.loc[mask, "source_household_id"].nunique() + + expected = { + kind: { + system: rounded_count( + sources( + (rows[_SUPPORT_ORDINAL] == ordinal) + & (rows["atomic_area_system"] == system) + ) + ) + for system in (EW, SCOT, NI) + } + for ordinal, kind in ((1, "spi_support_copy"), (2, "income_band_donor")) + } + flagged = { + system: rounded_count( + sources(rows[RFDR_COLUMN] & (rows["atomic_area_system"] == system)) + ) + for system in (EW, SCOT) + } + for clones in (1, 3): + evidence = copy_runs[clones]["evidence"]["copy"] + assert evidence["counted_among"] == "clone_0" + assert evidence["copy_source_households"] == expected + assert evidence["flagged_copy_source_households"] == flagged + assert evidence["no_flagged_copy"] == { + system: count == 0 for system, count in flagged.items() + } + assert evidence["level"] == "L3" + assert evidence["rfdr"]["roster"] == "pass" + assert evidence["validation"]["outcome"] == "pass" + assert evidence["brma"]["engine_households"] == "northern_ireland_copies" + assert evidence["brma"]["engine_simulations"] == "one" + assert evidence["rewrites"] == list(UK_SPI_COPY_AREAS_REWRITE_COLUMNS) + + +def test_the_copy_stage_counts_clone_zero_where_later_clones_copy_others( + copy_runs, +) -> None: + """Later clones copy other households, so a union of copied sources over + clones grows with K; the evidence counts clone 0's.""" + + household = copy_runs[3]["frames"]["copy"].table("household") + donors = _copies(household) & (household[_SUPPORT_ORDINAL] == 2).to_numpy() + zero = (household[_CLONE] == 0).to_numpy() + systems = household["atomic_area_system"].to_numpy(dtype=object) + sources = household["source_household_id"].to_numpy() + + def distinct(rows) -> dict[str, int]: + return { + system: len(set(sources[rows & (systems == system)].tolist())) + for system in (EW, SCOT, NI) + } + + # The invented frame tells the two counts apart. + assert distinct(donors & zero) != distinct(donors) + result = copy_runs[3]["results"]["copy"] + assert dict(result.by_kind["income_band_donor"]) == distinct(donors & zero) + + +def test_the_copy_stage_draws_with_g2s_definition_and_supports(runs, copy_runs) -> None: + g2 = runs[1]["evidence"]["g2"] + for clones in (1, 3): + copy = copy_runs[clones]["evidence"]["copy"] + for key in ("definition_sha256", "support_sha256", "stream", "seed"): + assert copy[key] == g2[key], key + assert copy["validation"] == g2["validation"] + + +# --- Evidence across clone counts --------------------------------------------------- + + +def test_g4_and_the_copy_stage_evidence_is_byte_equal_at_k_1_and_k_3( + copy_runs, +) -> None: + for stage in ("g4", "copy"): + single, triple = ({**copy_runs[clones]["evidence"][stage]} for clones in (1, 3)) + assert (single.pop("clone_count"), triple.pop("clone_count")) == (1, 3) + assert json.dumps(single, sort_keys=True) == json.dumps(triple, sort_keys=True) + + +def test_g4_and_the_copy_stage_evidence_prints_no_key_and_no_code(copy_runs) -> None: + household = copy_runs[3]["frames"]["copy"].table("household") + text = json.dumps(copy_runs[3]["evidence"], sort_keys=True) + for column in ( + IDENTITY_COLUMN, + "atomic_area_code", + "lsoa_code", + "local_authority_code", + "local_authority", + "brma", + ): + values = {str(value) for value in household[column]} + assert not any(value in text for value in values), column + + +def test_every_evidence_count_is_byte_equal_at_k_1_and_k_3(runs) -> None: + """The plan's "K clones count as one source": each stage's evidence is + the same bytes at K = 1 and K = 3 but for the clone count it states.""" + + for stage in ("g2", "g3", "g5"): + single, triple = ({**runs[clones]["evidence"][stage]} for clones in (1, 3)) + assert (single.pop("clone_count"), triple.pop("clone_count")) == (1, 3) + assert json.dumps(single, sort_keys=True) == json.dumps(triple, sort_keys=True) + + +def test_the_evidence_prints_no_key_and_no_code_of_a_household(runs) -> None: + household = _household(runs, 3) + text = json.dumps(runs[3]["evidence"], sort_keys=True) + for column in ( + IDENTITY_COLUMN, + *UK_SPINE_AREA_COLUMNS[:1], + *UK_SPINE_LADDER_COLUMNS, + ): + values = {str(value) for value in household[column]} + # The region codes the ladder writes are the regions' own GSS codes, + # which the evidence never names either. + assert not any(value in text for value in values), column diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_private_rent.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_private_rent.py new file mode 100644 index 0000000000..4f1693a2e5 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_private_rent.py @@ -0,0 +1,993 @@ +"""The clone-first spine's private-rent stage, G7 (microcosm#931). + +G7 reads each Great Britain private renter's rent off its BRMA's list of rents +with every row its own spine household, keyed on its geography key, tie-broken +on the stage's own stream. No stage declares it yet, so the transform here +runs as a stage built from the reviewed declaration spine_private_rent states, +on the committed lists of rents and census counts. + +The frames are invented households (toy rents, tenures and bedrooms), cloned +and keyed by the spine's own expansion and identity steps, placed by the +spine's area draw on test_support's toy conditioned supports, whose areas sit +in BRMAs the committed lists cover, given their BRMA through the committed +area-to-BRMA layer as the BRMA stage gives it, and given SPI copies keyed over +their copy path as the SPI-copy area stage keys them. + +They hold: on a copy-free one-clone frame the transform equals +``localise_rents`` given the same keys and stream; per-row keys run where the +dense line's spine keys refuse; clone 0's tie uniforms at K = 3 are K = 1's; +two FRS renters at one rent in one cell take distinct ranks; the sampled +rung's region floor counts true sources; Northern Ireland, other tenures and +zero rents keep their rents bit for bit; the re-draw does not depend on row +order; every refusal names no key or BRMA of a household; and the evidence, +counted among clone-0 rows, is byte-equal at K = 1 and K = 3. +""" + +from __future__ import annotations + +import functools +import hashlib +import json +from collections.abc import Callable, Mapping +from dataclasses import dataclass, replace +from importlib.resources import files + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.source_lineage import copy_paths +from microcosm.build.source_manifest import ( + SourceOperationSpec, + SourceStageSpec, + _reject_executable_parameter_keys, + _reject_incumbent_dependencies, +) +from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime import private_rent_localisation as prl +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.brma_area_layer import ( + area_brmas, + load_brma_area_layer, +) +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + ladder_clone_index_column, +) +from microcosm.build.uk_runtime.geographic_identity import ( + GEOGRAPHIC_IDENTITY_SOURCE, + frs_vintages_by_source_year, +) +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_SUPPORT_BRANCH, + UK_COPY_LINEAGE, + without_geographic_branch, +) +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + uk_time_period, +) +from microcosm.build.uk_runtime.property_localisation import spine_household_keys +from microcosm.build.uk_runtime.receipt_counts import rounded_count +from microcosm.build.uk_runtime.spine_geography import ( + ATOMIC_AREA_ASSIGN_STAGE_NAME, + UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS, + UKAtomicAreaAssignStageTransform, + atomic_area_assign_operation_parameters, +) +from microcosm.build.uk_runtime.spine_private_rent import ( + COUNTS_ARTIFACT_ROLE, + COUNTS_RESOURCE, + PRIVATE_RENT_OPERATION_KIND, + PRIVATE_RENT_STAGE_NAME, + QUANTILES_ARTIFACT_ROLE, + QUANTILES_RESOURCE, + RENT_RANK_PROCESS, + UK_PRIVATE_RENT_INPUTS, + UK_PRIVATE_RENT_MASS_CONSERVATION_REASON, + UK_PRIVATE_RENT_REWRITES, + UK_SPINE_PRIVATE_RENT_SEED, + UK_SPINE_PRIVATE_RENT_STREAM, + UK_SPINE_PRIVATE_RENT_STREAM_EXPERIMENT, + UKPrivateRentStageTransform, + draw_uk_spine_private_rents, + load_committed_private_rent_resources, + localise_uk_spine_private_rents, + private_rent_operation_parameters, + uk_spine_rent_tie_uniforms, +) +from microcosm.frame import Frame +from microcosm.graph.randomness import keyed_uniform +from test_support.microcosm_build.uk_conditioned_support_fixtures import ( + ToyHousehold, + toy_conditioned_support_payloads, + toy_spine_frame, +) + +_KEY = GEOGRAPHY_HOUSEHOLD_KEY_COLUMN +_CLONE = ladder_clone_index_column("household") +_SPI_FIELD = "spi_support_channel" +#: The design's stream for G7, spelled out here rather than read from the +#: module, so a drift in the module's constant fails. +_DESIGN_STREAM = ("sha256-u53-v1", "uk-spine-private-rent-v1", 0, 0) +#: An invented annual rent level; every toy rent is a factor of it. +_LEVEL = 14_000.0 +#: Northern Ireland's BRMA before the BRMA stage: the layer keeps it. +_NI_BRMA = "BELFAST" + + +@dataclass(frozen=True) +class _Home: + """An invented household's region and housing; its rent is ``factor`` + times the invented level, rounded to pence.""" + + region: str + tenure: str + bedrooms: int + factor: float + + @property + def rent(self) -> float: + return round(_LEVEL * self.factor, 2) + + +#: Invented households, one source each (position + 1). London holds two FRS +#: renters at one rent in one cell (0 and 1) and three households the stage +#: keeps (an owner, a social renter with a rent, a private renter with none); +#: every region with a re-drawn row holds FRS renters, so a full-scale spine +#: runs; Northern Ireland's private renter keeps its rent. +_HOMES: tuple[_Home, ...] = ( + _Home("LONDON", "RENT_PRIVATELY", 2, 1.0), + _Home("LONDON", "RENT_PRIVATELY", 2, 1.0), + _Home("LONDON", "RENT_PRIVATELY", 1, 0.8), + _Home("LONDON", "RENT_PRIVATELY", 3, 1.3), + _Home("LONDON", "RENT_PRIVATELY", 2, 0.9), + _Home("LONDON", "OWNED_OUTRIGHT", 3, 0.0), + _Home("LONDON", "RENT_FROM_COUNCIL", 2, 0.6), + _Home("LONDON", "RENT_PRIVATELY", 2, 0.0), + _Home("NORTH_EAST", "RENT_PRIVATELY", 2, 1.0), + _Home("NORTH_EAST", "RENT_PRIVATELY", 1, 0.9), + _Home("NORTH_EAST", "RENT_PRIVATELY", 3, 1.1), + _Home("NORTH_EAST", "OWNED_WITH_MORTGAGE", 3, 0.0), + _Home("YORKSHIRE", "RENT_PRIVATELY", 2, 1.0), + _Home("YORKSHIRE", "RENT_PRIVATELY", 4, 1.2), + _Home("WALES", "RENT_PRIVATELY", 2, 1.0), + _Home("SCOTLAND", "RENT_PRIVATELY", 2, 1.0), + _Home("SCOTLAND", "RENT_PRIVATELY", 1, 0.85), + _Home("NORTHERN_IRELAND", "RENT_PRIVATELY", 2, 0.55), + _Home("NORTHERN_IRELAND", "OWNED_OUTRIGHT", 3, 0.0), +) +#: The SPI copies (ordinal 1 in the support channel's field) of four +#: households, in every clone, with the housing the shell would draw for each +#: clone apart: London's 1-bedroom renter's copy rents four bedrooms, a cell +#: no London FRS renter holds; London's 2-bedroom renter's copy rents at a +#: rent of its clone's own; the North East owner's copy owns in clone 0 and +#: rents four bedrooms in every later clone; Scotland's copy rents one +#: bedroom. +_Copy = Callable[[int], _Home] +_COPIES: Mapping[int, _Copy] = { + 2: lambda clone: _Home("LONDON", "RENT_PRIVATELY", 4, 1.4), + 4: lambda clone: _Home("LONDON", "RENT_PRIVATELY", 2, 0.95 + 0.1 * clone), + 11: lambda clone: ( + _Home("NORTH_EAST", "OWNED_WITH_MORTGAGE", 3, 0.0) + if clone == 0 + else _Home("NORTH_EAST", "RENT_PRIVATELY", 4, 1.0 + 0.05 * clone) + ), + 15: lambda clone: _Home("SCOTLAND", "RENT_PRIVATELY", 1, 0.7), +} +#: Wales's FRS renter turned owner, with an SPI copy that rents: a region +#: whose only re-drawn renter is SPI-channel. +_WALES_OWNS = tuple( + _Home("WALES", "OWNED_OUTRIGHT", 2, 0.0) if position == 14 else home + for position, home in enumerate(_HOMES) +) +_WALES_COPY: Mapping[int, _Copy] = { + 14: lambda clone: _Home("WALES", "RENT_PRIVATELY", 2, 1.0), +} + + +# --- Building the frames ------------------------------------------------------------ + + +def _stage(**overrides: object) -> SourceStageSpec: + """G7's stage built from the reviewed declaration spine_private_rent + states, with ``overrides`` replacing fields.""" + + stage = SourceStageSpec( + stage=PRIVATE_RENT_STAGE_NAME, + survey="Family Resources Survey 2024-25", + source="Microcosm structural operator (microcosm#931)", + grain="household", + artifacts=tuple( + { + "role": role, + "kind": "public_aggregate_reference", + "resource": resource, + "format": "json", + "runtime_sha256_required": True, + } + for role, resource in ( + (QUANTILES_ARTIFACT_ROLE, QUANTILES_RESOURCE), + (COUNTS_ARTIFACT_ROLE, COUNTS_RESOURCE), + ) + ), + operations=tuple( + SourceOperationSpec(kind=kind, parameters=dict(parameters)) + for kind, parameters in private_rent_operation_parameters() + ), + outputs=(), + rewrites=UK_PRIVATE_RENT_REWRITES, + ) + return replace(stage, **overrides) + + +def _transform(*, sampled_rung: bool = False) -> UKPrivateRentStageTransform: + return UKPrivateRentStageTransform(stage=_stage(), sampled_rung=sampled_rung) + + +@functools.cache +def _area_draw() -> UKAtomicAreaAssignStageTransform: + """The spine's area draw (G2) on the toy conditioned supports.""" + + stage = SourceStageSpec( + stage=ATOMIC_AREA_ASSIGN_STAGE_NAME, + survey="Family Resources Survey 2024-25", + source="Microcosm structural operator (microcosm#931)", + grain="household", + artifacts=(), + operations=tuple( + SourceOperationSpec(kind=kind, parameters=dict(parameters)) + for kind, parameters in atomic_area_assign_operation_parameters() + ), + outputs=UK_ATOMIC_AREA_ASSIGN_OUTPUT_COLUMNS, + ) + return UKAtomicAreaAssignStageTransform( + stage=stage, supports=toy_conditioned_support_payloads() + ) + + +@functools.cache +def _layer(): + return load_brma_area_layer( + files("microcosm.build.uk").joinpath("brma_area_layer.json").read_bytes() + ) + + +@functools.cache +def _resources(): + return load_committed_private_rent_resources() + + +def _brmas(household: pd.DataFrame) -> np.ndarray: + """Each row's BRMA: off its drawn area in Great Britain, as the BRMA stage + reads it; Northern Ireland's kept.""" + + brma, _ = area_brmas( + layer=_layer(), + area_system=household["atomic_area_system"].to_numpy(dtype=object), + lsoa_code=household["lsoa_code"].to_numpy(dtype=object), + oa_code=household["oa_code"].to_numpy(dtype=object), + region=household["region"].to_numpy(dtype=object), + brma=np.full(len(household), _NI_BRMA, dtype=object), + ) + return brma + + +def _keys(household: pd.DataFrame, *, pooled: bool = False) -> pd.Series: + """Each row's geography key over its copy path, as the geography identity + and the SPI-copy area stage key them; ``pooled`` drops the geographic + branch, as the dense line's spine keys did.""" + + sources, paths = copy_paths( + household, UK_COPY_LINEAGE, require=(_SPI_FIELD, GEOGRAPHIC_SUPPORT_BRANCH) + ) + if pooled: + paths = without_geographic_branch(paths) + vintages = frs_vintages_by_source_year() + keys = [ + household_draw_key( + source=GEOGRAPHIC_IDENTITY_SOURCE, + source_vintage=vintages[int(year)], + source_household_id=int(source), + clone_path=path, + ) + for source, year, path in zip( + sources.tolist(), household["source_year"].tolist(), paths, strict=True + ) + ] + return pd.Series(keys, index=household.index, dtype=pd.StringDtype("python")) + + +def _house(household: pd.DataFrame, homes: list[_Home]) -> pd.DataFrame: + """``household`` with each row's tenure, bedrooms, rent and mortgage.""" + + housed = household.copy() + housed["tenure_type"] = pd.Series( + [home.tenure for home in homes], index=housed.index, dtype=object + ) + housed["num_bedrooms"] = np.asarray([home.bedrooms for home in homes], np.int64) + housed["rent"] = np.asarray([home.rent for home in homes], dtype=np.float64) + housed["mortgage_interest_repayment"] = np.asarray( + [3_000.0 if home.tenure == "OWNED_WITH_MORTGAGE" else 0.0 for home in homes] + ) + return housed + + +def _with_spi_copies(frame: Frame, copies: Mapping[int, _Copy]) -> Frame: + """``frame`` with an SPI copy of each named household in every clone, at + half its parent's weight, keyed over its copy path.""" + + household = frame.table("household") + person = frame.table("person") + benunit = frame.table("benunit") + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + positions = household["source_household_id"].to_numpy() - 1 + picked = np.isin(positions, list(copies)) + parents = household.loc[picked] + parent_ids = parents["household_id"].to_numpy() + homes = [ + copies[int(position)](int(clone)) + for position, clone in zip( + positions[picked], parents[_CLONE].to_numpy(), strict=True + ) + ] + copied = _house(parents, homes) + copied["household_id"] = copy_id_table.mint(parent_ids, _SPI_FIELD, 1) + copied["household_support_channel"] = "spi" + copied["household_support_clone_index"] = np.ones(len(copied), dtype=np.int64) + members = person.loc[person["person_household_id"].isin(parent_ids)].copy() + units = benunit.loc[benunit["benunit_id"].isin(members["person_benunit_id"])].copy() + for column in ("person_id", "person_household_id", "person_benunit_id"): + members[column] = copy_id_table.mint(members[column], _SPI_FIELD, 1) + units["benunit_id"] = copy_id_table.mint(units["benunit_id"], _SPI_FIELD, 1) + half = weights[picked] / 2.0 + weights[picked] = half + households = pd.concat([household, copied], ignore_index=True) + order = np.argsort(households["household_id"].to_numpy(), kind="stable") + households = households.iloc[order].reset_index(drop=True) + households[_KEY] = _keys(households) + return uk_national_frame( + person=pd.concat([person, members], ignore_index=True) + .sort_values("person_id", kind="stable") + .reset_index(drop=True), + benunit=pd.concat([benunit, units], ignore_index=True) + .sort_values("benunit_id", kind="stable") + .reset_index(drop=True), + household=households, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=np.concatenate([weights, half])[order], + mass_log=frame.mass_log, + ) + + +def _build(clones: int, homes: tuple[_Home, ...], copies: Mapping[int, _Copy]) -> Frame: + cases = [ToyHousehold(home.region, 0, "D", False, "L3") for home in homes] + placed = _area_draw()(toy_spine_frame(clones=clones, households=cases)) + household = placed.table("household") + positions = household["source_household_id"].to_numpy() - 1 + housed = _house(household, [homes[int(position)] for position in positions]) + housed["brma"] = _brmas(housed) + housed["household_support_channel"] = "frs" + housed["household_support_clone_index"] = np.zeros(len(housed), dtype=np.int64) + person = placed.table("person").copy() + first = ~person["person_household_id"].duplicated() + for column, amount in ( + ("housing_benefit_reported", 2_100.0), + ("universal_credit_reported", 4_200.0), + ): + person[column] = np.where(first, amount, 0.0) + frame = uk_national_frame( + person=person, + benunit=placed.table("benunit"), + household=housed, + time_period=uk_time_period(placed), + weight_kind=uk_household_weight_kind(placed), + household_weights=placed.weights_for("household").values, + mass_log=placed.mass_log, + ) + return _with_spi_copies(frame, copies) if copies else frame + + +@functools.cache +def _frame(clones: int, variant: str = "copies") -> Frame: + """The toy spine at G7's position: ``variant`` "copies" (the default), + "base" (no SPI copy) or "wales" (Wales's renter only an SPI copy).""" + + homes, copies = { + "copies": (_HOMES, _COPIES), + "base": (_HOMES, {}), + "wales": (_WALES_OWNS, _WALES_COPY), + }[variant] + return _build(clones, homes, copies) + + +def _household(clones: int, variant: str = "copies") -> pd.DataFrame: + return _frame(clones, variant).table("household") + + +def _weights(clones: int, variant: str = "copies") -> np.ndarray: + return np.asarray(_frame(clones, variant).weights_for("household").values) + + +def _draw(clones: int, variant: str = "copies", *, strict: bool = True): + return draw_uk_spine_private_rents( + _household(clones, variant), + _weights(clones, variant), + resources=_resources(), + strict=strict, + ) + + +def _redrawn(household: pd.DataFrame) -> np.ndarray: + return prl.redraw_mask( + household["region"].to_numpy(dtype=object), + household["tenure_type"].to_numpy(dtype=object), + household["rent"].to_numpy(dtype=float), + ) + + +def _localise(household, weights, *, spine_key, source, tie_uniform, strict=True): + """``localise_rents`` on ``household`` with the given keys, sources and + tie-breaks and the committed resources.""" + + return prl.localise_rents( + spine_key=np.asarray(spine_key, dtype=object), + source=np.asarray(source), + region=household["region"].to_numpy(dtype=object), + tenure_type=household["tenure_type"].to_numpy(dtype=object), + rent=household["rent"].to_numpy(dtype=float), + num_bedrooms=household["num_bedrooms"].to_numpy(), + brma=household["brma"].to_numpy(dtype=object), + channel=household["household_support_channel"].to_numpy(dtype=object), + weight=weights, + tie_uniform=tie_uniform, + quantiles=_resources().quantiles, + counts=_resources().counts, + strict=strict, + ) + + +def _ranks(household, weights, keys) -> pd.Series: + """Each re-drawn spine household's rank, as ``localise_rents`` ranks it + under ``keys``, with the stage's tie-breaks of those keys.""" + + redrawn = _redrawn(household) + spine = prl._spine_rows( + spine_key=np.asarray(keys, dtype=object)[redrawn], + source=household["source_household_id"].to_numpy()[redrawn], + region=household["region"].to_numpy(dtype=object)[redrawn], + category=prl.rent_categories(household["num_bedrooms"].to_numpy()[redrawn]), + rent=household["rent"].to_numpy(dtype=float)[redrawn], + channel=household["household_support_channel"].to_numpy(dtype=object)[redrawn], + weight=np.asarray(weights)[redrawn], + ) + ties = dict( + zip(spine.index, uk_spine_rent_tie_uniforms(list(spine.index)), strict=True) + ) + ranks, _ = prl.rent_uniforms(spine, ties) + return ranks + + +def _bits(values) -> np.ndarray: + return np.asarray(values, dtype=np.float64).view(np.int64) + + +# --- The reviewed declaration ------------------------------------------------------- + + +def test_the_reviewed_operations_are_declarative_and_name_the_stages_stream() -> None: + for kind, parameters in private_rent_operation_parameters(): + _reject_executable_parameter_keys(parameters, context=kind) + _reject_incumbent_dependencies(parameters, context=kind) + assert json.loads(json.dumps(parameters)) == parameters + (kind, rents), (receipt_kind, receipt) = private_rent_operation_parameters() + assert kind == PRIVATE_RENT_OPERATION_KIND == "localise_private_rents" + assert UK_SPINE_PRIVATE_RENT_STREAM == _DESIGN_STREAM + assert ( + rents["stream"] == UK_SPINE_PRIVATE_RENT_STREAM_EXPERIMENT == _DESIGN_STREAM[1] + ) + assert rents["seed"] == UK_SPINE_PRIVATE_RENT_SEED == _DESIGN_STREAM[3] == 0 + assert rents["identity"] == _KEY and rents["process"] == RENT_RANK_PROCESS + # The stage's stream is its own, not the retired dense node's. + assert UK_SPINE_PRIVATE_RENT_STREAM_EXPERIMENT != prl.STREAM_EXPERIMENT + assert rents["sampled_region_renters"] == prl.MIN_SAMPLED_REGION_RENTERS + assert rents["scale_band"] == list(prl.SCALE_BAND) + assert rents["weeks_in_year"] == prl.WEEKS_IN_YEAR + assert rents["method"] == prl.METHOD_VERSION + assert receipt_kind == "record_mass_conservation_receipt" + assert receipt["reason"] == UK_PRIVATE_RENT_MASS_CONSERVATION_REASON + assert receipt["declared_factor"] == 1.0 and receipt["entity"] == "household" + + +def test_the_transform_runs_only_as_its_reviewed_stage() -> None: + stage = _stage() + UKPrivateRentStageTransform(stage=stage) + rents, receipt = stage.operations + drifted = SourceOperationSpec( + kind=rents.kind, parameters={**rents.parameters, "seed": 1} + ) + (quantiles, counts) = stage.artifacts + for overrides, message in ( + ({"stage": "frs_brma"}, "Expected stage"), + ({"grain": "benunit"}, "grain must be 'household'"), + ({"outputs": ("rent",)}, "rewrites exactly the rent"), + ({"rewrites": ("rent", "subrent")}, "rewrites exactly the rent"), + ({"rewrites": ()}, "rewrites exactly the rent"), + ({"operations": (drifted, receipt)}, "drifted"), + ({"operations": (rents,)}, "operation order drifted"), + ({"artifacts": (quantiles,)}, "and no other"), + ( + {"artifacts": (quantiles, {**counts, "runtime_sha256_required": False})}, + "runtime_sha256_required", + ), + ( + { + "artifacts": ( + quantiles, + counts, + { + "role": "other", + "resource": "x.json", + "runtime_sha256_required": True, + }, + ) + }, + "and no other", + ), + ): + with pytest.raises(ValueError, match=message): + UKPrivateRentStageTransform(stage=_stage(**overrides)) + with pytest.raises(TypeError, match="SourceStageSpec"): + UKPrivateRentStageTransform(stage=object()) + with pytest.raises(TypeError, match="a bool"): + UKPrivateRentStageTransform(stage=stage, sampled_rung="no") + with pytest.raises(TypeError, match="a bool"): + localise_uk_spine_private_rents( + _frame(1), resources=_resources(), sampled_rung=1 + ) + + +def test_the_transform_reads_the_committed_resources_and_needs_a_run() -> None: + transform = _transform() + package = files("microcosm.build.uk") + assert ( + transform.resources.quantiles_sha256 + == hashlib.sha256(package.joinpath(QUANTILES_RESOURCE).read_bytes()).hexdigest() + ) + assert ( + transform.resources.counts_sha256 + == hashlib.sha256(package.joinpath(COUNTS_RESOURCE).read_bytes()).hexdigest() + ) + assert transform.output_columns() == () + with pytest.raises(RuntimeError, match="completed stage run"): + transform.checkpoint_metadata() + with pytest.raises(TypeError, match="Frame"): + transform(_household(1)) + + +# --- G7 against localise_rents ------------------------------------------------------ + + +def test_a_copy_free_one_clone_frame_equals_localise_rents_on_the_same_keys() -> None: + frame = _frame(1, "base") + household = frame.table("household") + assert set(household["household_support_channel"]) == {"frs"} + rents = _transform()(frame).table("household")["rent"] + keys = household[_KEY].to_numpy(dtype=object) + redrawn = _redrawn(household) + ties = keyed_uniform( + stream=_DESIGN_STREAM, keys=[(key, "rent_rank") for key in keys[redrawn]] + ) + expected, details = _localise( + household, + _weights(1, "base"), + spine_key=keys, + source=household["source_household_id"].to_numpy(), + tie_uniform=dict(zip(keys[redrawn], ties.tolist(), strict=True)), + ) + assert np.array_equal(_bits(rents), _bits(expected)) + assert details["spine_households_redrawn"] == details["rows_redrawn"] > 0 + # Every Great Britain private renter with a rent moved. + assert (rents.to_numpy()[redrawn] != household["rent"].to_numpy()[redrawn]).all() + + +def test_per_row_keys_run_where_the_dense_lines_spine_keys_refuse() -> None: + """The dense node's spine keys read the CGT lineage, which the spine + writes only after G7, and pool the clones of an SPI copy whose housing the + shell drew for each clone apart; G7's per-row keys run on both.""" + + household = _household(1) + with pytest.raises(ValueError, match="lacks lineage"): + spine_household_keys( + household, + source=GEOGRAPHIC_IDENTITY_SOURCE, + source_vintage=frs_vintages_by_source_year()[ + int(household["source_year"].iloc[0]) + ], + ) + _transform()(_frame(1)) + two = _household(2) + pooled = _keys(two, pooled=True) + with pytest.raises(ValueError, match="disagree"): + _localise( + two, + _weights(2), + spine_key=pooled.to_numpy(dtype=object), + source=two["source_household_id"].to_numpy(), + tie_uniform={key: 0.5 for key in pooled}, + ) + out = _transform()(_frame(2)).table("household") + redrawn = _redrawn(two) + assert (out["rent"].to_numpy()[redrawn] != two["rent"].to_numpy()[redrawn]).all() + + +def test_clone_zero_takes_k_1s_tie_uniforms_at_k_3() -> None: + single, triple = _household(1), _household(3) + one, three = _draw(1), _draw(3) + zero = triple[_CLONE].to_numpy() == 0 + # Clone 0's rows are K = 1's rows: the same ids and keys. + assert triple.loc[zero, "household_id"].tolist() == single["household_id"].tolist() + assert triple.loc[zero, _KEY].tolist() == single[_KEY].tolist() + + def by_id(draw, household) -> pd.Series: + ids = household.loc[draw.tie_uniforms.index, "household_id"].to_numpy() + return pd.Series(draw.tie_uniforms.to_numpy(), index=ids) + + k1, k3 = by_id(one, single), by_id(three, triple) + assert np.array_equal(_bits(k3.loc[k1.index]), _bits(k1)) + # The other clones draw their own. + others = k3.drop(index=k1.index) + assert len(others) and not set(others.tolist()) & set(k1.tolist()) + # Ranks and scales pool over clones, so clone 0's rents nest approximately. + rents_k3 = pd.Series(three.rent.to_numpy(), index=triple["household_id"].to_numpy()) + np.testing.assert_allclose( + rents_k3.loc[single["household_id"].to_numpy()].to_numpy(), + one.rent.to_numpy(), + rtol=1e-9, + ) + + +def test_two_frs_renters_at_one_rent_in_one_cell_take_distinct_ranks() -> None: + household, weights = _household(1, "base"), _weights(1, "base") + a, b = household.index[household["source_household_id"].isin([1, 2])] + assert household.loc[a, "rent"] == household.loc[b, "rent"] + assert household.loc[a, "num_bedrooms"] == household.loc[b, "num_bedrooms"] + ranks = _ranks(household, weights, household[_KEY]) + first, second = ranks[household.loc[a, _KEY]], ranks[household.loc[b, _KEY]] + assert first != second + # Both sit inside the one weight step their shared rent spans. + redrawn = _redrawn(household) + cell = ( + redrawn + & (household["region"] == "LONDON").to_numpy() + & (household["num_bedrooms"] == 2).to_numpy() + ) + rent, total = household.loc[a, "rent"], weights[cell].sum() + below = weights[cell & (household["rent"] < rent).to_numpy()].sum() / total + at_or_below = weights[cell & (household["rent"] <= rent).to_numpy()].sum() / total + for rank in (first, second): + assert below <= rank <= at_or_below + # At K = 2 the two clones of one renter take ranks of their own; keyed on + # the path without the geographic branch, as the dense line keyed them, + # they would share one. + two, two_weights = _household(2, "base"), _weights(2, "base") + clones = two.index[two["source_household_id"] == 1] + per_row = _ranks(two, two_weights, two[_KEY]) + assert len({per_row[two.loc[row, _KEY]] for row in clones}) == 2 + pooled = _keys(two, pooled=True) + assert len({pooled[row] for row in clones}) == 1 + assert len(_ranks(two, two_weights, pooled)) == len(per_row) // 2 + + +def test_the_sampled_rungs_region_floor_counts_true_sources() -> None: + """Every toy region holds fewer FRS renter sources than + ``MIN_SAMPLED_REGION_RENTERS`` (a statistical floor, not the disclosure + floor), so a sampled rung ranks every cell against Great Britain at every + K. London's five FRS renters are fifteen rows at K = 3: counted by row, + London would rank on its own.""" + + renters = _household(1) + frs = _redrawn(renters) & (renters["household_support_channel"] == "frs").to_numpy() + held = renters.loc[frs].groupby("region")["source_household_id"].nunique() + assert held.max() < prl.MIN_SAMPLED_REGION_RENTERS + assert held["LONDON"] * 3 >= prl.MIN_SAMPLED_REGION_RENTERS + for clones in (1, 3): + draw = _draw(clones, strict=False) + assert draw.cells_ranked_against_their_region == 0 + assert draw.cells_ranked_against_great_britain > 0 + household, weights = _household(3), _weights(3) + draw = _draw(3, strict=False) + keys = household[_KEY].to_numpy(dtype=object) + redrawn = _redrawn(household) + ties = dict( + zip( + keys[redrawn], + uk_spine_rent_tie_uniforms(list(keys[redrawn])).tolist(), + strict=True, + ) + ) + by_source, sourced = _localise( + household, + weights, + spine_key=keys, + source=household["source_household_id"].to_numpy(), + tie_uniform=ties, + strict=False, + ) + by_row, rowed = _localise( + household, + weights, + spine_key=keys, + source=household["household_id"].to_numpy(), + tie_uniform=ties, + strict=False, + ) + assert np.array_equal(_bits(draw.rent), _bits(by_source)) + assert rowed["great_britain_cells"] < sourced["great_britain_cells"] + london = redrawn & (household["region"] == "LONDON").to_numpy() + assert not np.array_equal(by_row[london], by_source[london]) + + +def test_a_full_scale_spine_refuses_a_region_without_frs_renters() -> None: + """Wales's only re-drawn renter is an SPI copy: a full-scale spine + refuses, and a sampled rung ranks and scales Wales on Great Britain's.""" + + frame = _frame(1, "wales") + with pytest.raises(ValueError, match="No FRS private renter"): + _transform()(frame) + transform = _transform(sampled_rung=True) + out = transform(frame).table("household") + wales = (out["region"] == "WALES").to_numpy() & _redrawn(frame.table("household")) + assert wales.any() + assert transform.checkpoint_metadata()["evidence"]["strict"] is False + + +def test_northern_ireland_other_tenures_and_zero_rents_keep_their_rents() -> None: + frame = _frame(3) + transform = _transform() + out = transform(frame) + before, after = frame.table("household"), out.table("household") + redrawn = _redrawn(before) + kept = ~redrawn + assert np.array_equal( + _bits(after["rent"].to_numpy()[kept]), _bits(before["rent"].to_numpy()[kept]) + ) + held = before.loc[kept] + assert (held["region"] == "NORTHERN_IRELAND").any() + assert ((held["tenure_type"] == "RENT_PRIVATELY") & (held["rent"] > 0)).any() + assert (held["tenure_type"] == "RENT_FROM_COUNCIL").any() + assert ((held["tenure_type"] == "RENT_PRIVATELY") & (held["rent"] == 0)).any() + assert ( + (held["household_support_channel"] == "spi") + & (held["tenure_type"] == "OWNED_WITH_MORTGAGE") + ).any() + assert after["rent"].dtype == before["rent"].dtype == np.float64 + assert after.drop(columns="rent").equals(before.drop(columns="rent")) + # Reported Housing Benefit, Universal Credit and mortgages, and every + # other table, stay as they were. + for entity in ("person", "benunit"): + assert out.table(entity).equals(frame.table(entity)) + assert np.array_equal( + out.weights_for("household").values, frame.weights_for("household").values + ) + assert out.mass_log[:-1] == frame.mass_log + receipt = out.mass_log[-1] + assert receipt.reason == UK_PRIVATE_RENT_MASS_CONSERVATION_REASON + assert receipt.old_total == receipt.new_total and receipt.declared_factor == 1.0 + + +def test_the_rents_do_not_depend_on_row_order() -> None: + household, weights = _household(3), _weights(3) + order = np.random.default_rng(931).permutation(len(household)) + shuffled = household.iloc[order] + draw = _draw(3) + permuted = draw_uk_spine_private_rents( + shuffled, weights[order], resources=_resources(), strict=True + ) + assert permuted.rent.index.equals(shuffled.index) + assert np.array_equal(_bits(permuted.rent.loc[household.index]), _bits(draw.rent)) + assert np.array_equal( + _bits(permuted.tie_uniforms.loc[draw.tie_uniforms.index]), + _bits(draw.tie_uniforms), + ) + for name in ( + "source_households_redrawn", + "cells_ranked_against_their_region", + "cells_ranked_against_great_britain", + ): + assert getattr(permuted, name) == getattr(draw, name) + + +# --- Evidence ----------------------------------------------------------------------- + + +def test_the_evidence_counts_clone_zero_sources_and_cells() -> None: + transform = _transform() + transform(_frame(1)) + evidence = transform.checkpoint_metadata()["evidence"] + household = _household(1) + redrawn = _redrawn(household) + sources = household.loc[redrawn, "source_household_id"].nunique() + assert evidence == { + "stage": PRIVATE_RENT_STAGE_NAME, + "clone_count": 1, + "counted_among": "clone_0", + "source_households_redrawn": rounded_count(sources), + # London's four-bedroom cell, where only an SPI copy rents. + "cells_ranked_against_their_region": 1, + "cells_ranked_against_great_britain": 0, + "strict": True, + "seed": 0, + "stream": list(_DESIGN_STREAM), + "quantiles_sha256": transform.resources.quantiles_sha256, + "counts_sha256": transform.resources.counts_sha256, + "method": prl.METHOD_VERSION, + "weights": "unchanged", + "rewrites": ["rent"], + } + # At K = 1 every row is clone 0: the cells are those localise_rents + # routes, on a full-scale spine and on a sampled rung. + keys = household[_KEY].to_numpy(dtype=object) + ties = dict( + zip( + keys[redrawn], + uk_spine_rent_tie_uniforms(list(keys[redrawn])).tolist(), + strict=True, + ) + ) + for strict in (True, False): + _, details = _localise( + household, + _weights(1), + spine_key=keys, + source=household["source_household_id"].to_numpy(), + tie_uniform=ties, + strict=strict, + ) + draw = _draw(1, strict=strict) + assert ( + draw.cells_ranked_against_their_region, + draw.cells_ranked_against_great_britain, + ) == (details["fallback_cells"], details["great_britain_cells"]) + + +def test_the_evidence_counts_clone_zero_where_later_clones_differ() -> None: + """The North East owner's SPI copy rents four bedrooms in clones 1 and 2 + only, so the re-drawn sources and the cells over every clone grow with K; + the evidence counts clone 0.""" + + household, weights = _household(3), _weights(3) + draw = _draw(3) + redrawn = _redrawn(household) + zero = (household[_CLONE] == 0).to_numpy() + sources = household["source_household_id"].to_numpy() + in_clone_zero = len(set(sources[redrawn & zero].tolist())) + in_every_clone = len(set(sources[redrawn].tolist())) + assert in_every_clone > in_clone_zero + assert draw.source_households_redrawn == in_clone_zero + keys = household[_KEY].to_numpy(dtype=object) + _, details = _localise( + household, + weights, + spine_key=keys, + source=sources, + tie_uniform=dict( + zip( + keys[redrawn], + uk_spine_rent_tie_uniforms(list(keys[redrawn])).tolist(), + strict=True, + ) + ), + ) + assert details["fallback_cells"] > draw.cells_ranked_against_their_region == 1 + + +@pytest.mark.parametrize("sampled_rung", [False, True]) +def test_every_evidence_count_is_byte_equal_at_k_1_and_k_3(sampled_rung) -> None: + """The plan's "K clones count as one source": the evidence is the same + bytes at K = 1 and K = 3 but for the clone count it states.""" + + evidence = {} + for clones in (1, 3): + transform = _transform(sampled_rung=sampled_rung) + transform(_frame(clones)) + evidence[clones] = dict(transform.checkpoint_metadata()["evidence"]) + assert (evidence[1].pop("clone_count"), evidence[3].pop("clone_count")) == (1, 3) + assert json.dumps(evidence[1], sort_keys=True) == json.dumps( + evidence[3], sort_keys=True + ) + + +def test_the_evidence_prints_no_key_and_no_brma_of_a_household() -> None: + transform = _transform() + transform(_frame(3)) + text = json.dumps(transform.checkpoint_metadata(), sort_keys=True) + household = _household(3) + for column in (_KEY, "brma", "atomic_area_code", "lsoa_code"): + values = {str(value) for value in household[column]} + assert not any(value in text for value in values), column + + +# --- Refusals ----------------------------------------------------------------------- + + +@pytest.mark.parametrize("column", UK_PRIVATE_RENT_INPUTS) +def test_a_missing_input_column_is_refused_by_name(column) -> None: + with pytest.raises(ValueError, match="frame lacks them") as refused: + draw_uk_spine_private_rents( + _household(1).drop(columns=column), + _weights(1), + resources=_resources(), + strict=True, + ) + assert column in str(refused.value) + + +def test_a_rent_other_than_float64_and_misaligned_weights_are_refused() -> None: + household = _household(1) + narrow = household.assign(rent=household["rent"].astype("float32")) + with pytest.raises(ValueError, match="float64"): + draw_uk_spine_private_rents( + narrow, _weights(1), resources=_resources(), strict=True + ) + with pytest.raises(ValueError, match="one household weight"): + draw_uk_spine_private_rents( + household, _weights(1)[:-1], resources=_resources(), strict=True + ) + + +def _refusal(household: pd.DataFrame) -> str: + with pytest.raises(ValueError) as refused: + draw_uk_spine_private_rents( + household, _weights(1), resources=_resources(), strict=True + ) + return str(refused.value) + + +def test_a_repeated_or_missing_key_among_re_drawn_rows_is_refused_naming_none() -> None: + """An SPI copy keyed as its parent (the SPI-copy area stage never ran) + shares its key; a re-drawn row without a key has none.""" + + household = _household(1) + redrawn = _redrawn(household) + copy = household.index[ + redrawn & (household["household_support_channel"] == "spi").to_numpy() + ][0] + (parent,) = household.index[ + (household["household_support_channel"] == "frs").to_numpy() + & ( + household["source_household_id"] + == household.loc[copy, "source_household_id"] + ).to_numpy() + ] + assert redrawn[household.index.get_loc(parent)] + shared = household.copy() + shared.loc[copy, _KEY] = household.loc[parent, _KEY] + message = _refusal(shared) + assert "share one" in message + assert str(household.loc[parent, _KEY]) not in message + missing = household.copy() + missing.loc[copy, _KEY] = pd.NA + assert "needs its" in _refusal(missing) + # A row the stage does not re-draw needs no key of its own. + owner = household.index[~redrawn][0] + unkeyed = household.copy() + unkeyed.loc[owner, _KEY] = pd.NA + draw_uk_spine_private_rents( + unkeyed, _weights(1), resources=_resources(), strict=True + ) + + +@pytest.mark.parametrize("brma", ["MARS_AND_VENUS", None]) +def test_a_brma_without_a_list_is_refused_naming_no_brma(brma) -> None: + household = _household(1) + row = household.index[_redrawn(household)][0] + placed = household.copy() + placed.loc[row, "brma"] = brma + message = _refusal(placed) + assert "no list of rents" in message + assert "MARS" not in message + for value in set(household["brma"]): + assert str(value) not in message diff --git a/test_support/microcosm_build/uk_conditioned_support_fixtures.py b/test_support/microcosm_build/uk_conditioned_support_fixtures.py new file mode 100644 index 0000000000..c85d4a04b3 --- /dev/null +++ b/test_support/microcosm_build/uk_conditioned_support_fixtures.py @@ -0,0 +1,472 @@ +"""Toy UK conditioned atomic-area supports over public area codes (microcosm#931). + +The clone-first spine's area stages draw each household's area from the three +conditioned supports and then read committed resources keyed by the codes +the draw gives it: the local-authority stage the names roster, the BRMA stage +the area-to-BRMA layer, the fuel-relief stage the RFDR layer, the private-rent +stage the BRMAs' lists of rents and the home-value stage the constituency +knots. These toy supports carry, for all three systems, codes those resources +hold, so every stage resolves against the committed resources themselves: + +- each toy area's local authority is on ``local_authority_names.json``; +- its England-and-Wales LSOA, or its Scottish output area, is on + ``brma_area_layer.json`` in the area's own FRS region, under a BRMA that + ``brma_rent_quantiles.json`` lists rents for; +- its England-and-Wales constituency has knots in + ``high_value_home_value_knots.json``; +- two toy areas are output areas ``rfdr_area_layer.json`` flags: one of the + Hawes output areas in Yorkshire, and one Highlands-and-Islands output area + in Scotland. Every household drawn into their cells lands in them. + +The other codes (England and Wales's other output areas, MSOAs, wards, +ITL3s, Scottish zones and constituencies, Northern Ireland's zones, electoral +areas and constituencies) are well-formed codes of the area's nation. Which codes sit +together in one toy area is chosen for the tests, not read from the ONS +lookups, and every count, decile and band weight is invented. + +The fuel-relief stage refuses a layer that flags an output area outside its +system's support. So, unless a test asks for the toy areas alone, each Great +Britain support also carries every other output area the committed RFDR +layer flags, as a filler with no households, never drawn, on its system's +flagged toy area's codes. + +:data:`TOY_HOUSEHOLDS` are invented households, each with the fallback level +the toy supports give it; :func:`toy_spine_frame` builds them into a frame of +``K`` geographic clones keyed as the spine keys them, through the spine's own +expansion and identity steps. +""" + +from __future__ import annotations + +import functools +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from importlib.resources import files + +import numpy as np +import pandas as pd + +from microcosm.build.atomic_geography import AtomicSupport, decode_atomic_support +from microcosm.build.uk_runtime.atomic_area_support import ( + BAND_WEIGHT_COLUMNS, + BAND_WEIGHT_SCALE, + IMD_DECILE_COLUMN, + SYSTEMS, + assemble_uk_atomic_area_support, + assemble_uk_conditioned_area_support, +) +from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE +from microcosm.frame import Frame +from test_support.microcosm_build.uk_clone_first_frames import base_id_tables + +__all__ = [ + "EW", + "NI", + "SCOT", + "TOY_AREAS", + "TOY_FLAGGED_AREAS", + "TOY_HOUSEHOLDS", + "ToyArea", + "ToyHousehold", + "committed_rfdr_areas", + "toy_conditioned_support_payloads", + "toy_conditioned_supports", + "toy_household_table", + "toy_spine_frame", +] + +EW, SCOT, NI = SYSTEMS + + +@dataclass(frozen=True) +class ToyArea: + """One toy area of a support: its codes, its decile and its households. + + ``bands`` gives the households in each council-tax band, which sum to the + area's households; Northern Ireland's areas give ``households`` instead. + """ + + system: str + region: str + oa_code: str + lsoa_code: str + msoa_code: str + local_authority_code: str + ward_code: str + constituency_code: str + itl3_code: str + imd_decile: str + bands: Mapping[str, int] = field(default_factory=dict) + households: int = 0 + + def household_count(self) -> int: + return sum(dict(self.bands).values()) if self.bands else int(self.households) + + +def _ew(region, oa, lsoa, msoa, la, ward, pcon, itl3, decile, **bands) -> ToyArea: + return ToyArea(EW, region, oa, lsoa, msoa, la, ward, pcon, itl3, decile, bands) + + +def _scot(oa, lsoa, msoa, la, ward, pcon, itl3, decile, **bands) -> ToyArea: + return ToyArea( + SCOT, "SCOTLAND", oa, lsoa, msoa, la, ward, pcon, itl3, decile, bands + ) + + +def _ni(zone, super_zone, district, dea, pcon, itl3, decile, households) -> ToyArea: + return ToyArea( + NI, + "NORTHERN_IRELAND", + zone, + zone, + super_zone, + district, + dea, + pcon, + itl3, + decile, + households=households, + ) + + +#: The toy areas. Each region has household mass; London's cells give every +#: fallback level a household (decile 1 holds no band H, decile 9 no area), +#: Wales holds the only band I, and the North East no band G or H at all. +# fmt: off +TOY_AREAS: tuple[ToyArea, ...] = ( + _ew("NORTH_EAST", "E00042101", "E01008162", "E02001700", "E08000021", "E05001101", + "E14001063", "TLC22", "3", A=20, B=8, C=4, D=2), + _ew("NORTH_EAST", "E00042102", "E01008163", "E02001700", "E08000021", "E05001101", + "E14001063", "TLC22", "7", A=4, B=6, C=8, D=6, E=4, F=2), + _ew("NORTH_WEST", "E00025101", "E01004969", "E02001001", "E08000003", "E05000801", + "E14001064", "TLD33", "1", A=15, B=10, C=5), + _ew("YORKSHIRE", "E00057101", "E01010646", "E02002301", "E08000035", "E05001401", + "E14001065", "TLE42", "5", B=10, C=10, D=10), + # A Hawes output area, which the RFDR layer flags. + _ew("YORKSHIRE", "E00141336", "E01027587", "E02005701", "E06000065", "E05006001", + "E14001066", "TLE22", "6", C=2, D=3, E=3, F=2), + _ew("EAST_MIDLANDS", "E00069101", "E01013810", "E02002801", "E06000018", "E05001701", + "E14001067", "TLF14", "3", A=10, B=10, C=5, D=5), + _ew("WEST_MIDLANDS", "E00045101", "E01008881", "E02001801", "E08000025", "E05001201", + "E14001068", "TLG31", "2", A=10, B=15, C=5), + _ew("EAST_OF_ENGLAND", "E00092101", "E01017943", "E02003701", "E07000008", + "E05002701", "E14001069", "TLH12", "9", C=5, D=10, E=10, F=5), + _ew("LONDON", "E00000001", "E01000001", "E02000001", "E09000001", "E05009301", + "E14001070", "TLI31", "1", A=5, B=10, C=10, D=10, E=5), + _ew("LONDON", "E00000003", "E01000002", "E02000001", "E09000001", "E05009302", + "E14001070", "TLI31", "1", C=5, D=10, E=10, F=5), + _ew("LONDON", "E00000005", "E01000003", "E02000001", "E09000033", "E05013801", + "E14001071", "TLI32", "4", E=5, F=5, G=10, H=20), + _ew("LONDON", "E00009001", "E01001722", "E02000801", "E09000030", "E05009401", + "E14001072", "TLI42", "2", A=10, B=20, C=10), + _ew("SOUTH_EAST", "E00085101", "E01016849", "E02003501", "E06000043", "E05002401", + "E14001073", "TLJ21", "6", B=5, C=10, D=10, E=5), + _ew("SOUTH_WEST", "E00073101", "E01014395", "E02003001", "E06000023", "E05001901", + "E14001074", "TLK11", "5", A=10, B=10, C=10), + _ew("WALES", "W00009001", "W01001694", "W02000401", "W06000015", "W05001001", + "W07000081", "TLL22", "2", A=5, B=5, C=10, D=5, I=5), + _ew("WALES", "W00009002", "W01001695", "W02000401", "W06000015", "W05001002", + "W07000082", "TLL22", "8", D=10, E=10, F=5, G=3, H=2), + _scot("S00141165", "S01008417", "S02001575", "S12000036", "S13002922", "S14000021", + "TLM75", "2", A=10, B=10, C=10), + _scot("S00141166", "S01008418", "S02001575", "S12000036", "S13002923", "S14000021", + "TLM75", "8", D=10, E=10, F=5, G=3, H=2), + # A Highlands-and-Islands output area, which the RFDR layer flags. + _scot("S00162038", "S01010001", "S02001900", "S12000013", "S13002608", "S14000022", + "TLM64", "4", A=3, B=4, C=3), + _ni("N20000001", "N21000001", "N09000003", "N10000301", "N05000001", "TLN06", "1", 30), + _ni("N20000002", "N21000001", "N09000003", "N10000301", "N05000001", "TLN06", "1", 20), + _ni("N20000100", "N21000020", "N09000011", "N10001101", "N05000002", "TLN09", "6", 25), +) +# fmt: on +#: The toy areas the committed RFDR layer flags, by system: the hosts of each +#: system's filler rows. +TOY_FLAGGED_AREAS: Mapping[str, str] = {EW: "E00141336", SCOT: "S00162038"} + + +@dataclass(frozen=True) +class ToyHousehold: + """An invented household's draw inputs and the level the toys give it.""" + + region: str + imd_decile: int + council_tax_band: str + council_tax_band_observed: bool + level: str + + +def _household(region, decile, band, observed, level) -> ToyHousehold: + return ToyHousehold(region, decile, band, observed, level) + + +#: Invented households, one per case: L0 where the band has mass in the +#: household's decile cell, L1 where only households do (band H in London's +#: decile 1, band A in Wales's decile 8, no band observed), L2 where the decile +#: is missing or has no area and the band has mass in the region, L3 where +#: neither input holds (no band G or H in the North East). Northern Ireland +#: weighs no band: L1 or L3. Two households land in the flagged toy areas. +TOY_HOUSEHOLDS: tuple[ToyHousehold, ...] = ( + _household("LONDON", 1, "B", True, "L0"), + _household("LONDON", 1, "H", True, "L1"), + _household("LONDON", 0, "H", True, "L2"), + _household("LONDON", 0, "D", False, "L3"), + _household("LONDON", 9, "C", True, "L2"), + _household("LONDON", 4, "G", False, "L1"), + _household("LONDON", 2, "A", True, "L0"), + _household("NORTH_EAST", 3, "H", True, "L1"), + _household("NORTH_EAST", 0, "H", True, "L3"), + _household("NORTH_EAST", 7, "E", True, "L0"), + _household("NORTH_WEST", 1, "A", True, "L0"), + _household("YORKSHIRE", 5, "B", True, "L0"), + _household("YORKSHIRE", 6, "D", True, "L0"), + _household("EAST_MIDLANDS", 3, "C", True, "L0"), + _household("WEST_MIDLANDS", 0, "B", False, "L3"), + _household("EAST_OF_ENGLAND", 9, "D", True, "L0"), + _household("SOUTH_EAST", 6, "E", True, "L0"), + _household("SOUTH_WEST", 5, "C", True, "L0"), + _household("WALES", 2, "I", True, "L0"), + _household("WALES", 8, "A", True, "L1"), + _household("SCOTLAND", 2, "A", True, "L0"), + _household("SCOTLAND", 4, "C", True, "L0"), + _household("SCOTLAND", 0, "H", True, "L2"), + _household("SCOTLAND", 3, "B", False, "L3"), + _household("NORTHERN_IRELAND", 1, "D", False, "L1"), + _household("NORTHERN_IRELAND", 3, "D", False, "L3"), + _household("NORTHERN_IRELAND", 0, "D", False, "L3"), + _household("NORTHERN_IRELAND", 6, "D", False, "L1"), +) + +_CODE_COLUMNS = ( + "oa_code", + "constituency_code", + "region_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "itl3_code", +) + + +def committed_rfdr_areas() -> dict[str, tuple[str, ...]]: + """The output areas the committed RFDR layer flags, by system, sorted.""" + + payload = json.loads( + files("microcosm.build.uk").joinpath("rfdr_area_layer.json").read_bytes() + ) + return {system: tuple(sorted(codes)) for system, codes in payload["areas"].items()} + + +def _filler(host: ToyArea, oa_code: str) -> ToyArea: + """A flagged output area with no households, on its host's other codes.""" + + return ToyArea( + host.system, + host.region, + oa_code, + host.lsoa_code, + host.msoa_code, + host.local_authority_code, + host.ward_code, + host.constituency_code, + host.itl3_code, + host.imd_decile, + households=0, + ) + + +def _areas(system: str, *, rfdr_roster: bool) -> list[ToyArea]: + areas = [area for area in TOY_AREAS if area.system == system] + if rfdr_roster and system in TOY_FLAGGED_AREAS: + (host,) = [area for area in areas if area.oa_code == TOY_FLAGGED_AREAS[system]] + held = {area.oa_code for area in areas} + areas += [ + _filler(host, code) + for code in committed_rfdr_areas()[system] + if code not in held + ] + return areas + + +def _base_metadata(system: str) -> dict[str, dict[str, str]]: + vintage = "2022_census" if system == SCOT else "2021_census" + metadata = {} + for column in (*_CODE_COLUMNS, "population", "households"): + if column in {"population", "households"}: + metadata[column] = { + "kind": "weight", + "source": f"toy-{system}-{column}", + "basis": f"toy {column}", + } + continue + metadata[column] = { + "kind": "code", + "source": f"toy-{system}-{column}", + "vintage": vintage, + "relation": "exact" + if column in {"oa_code", "lsoa_code", "msoa_code", "region_code"} + else "best_fit", + } + metadata["constituency_code"]["vintage"] = "2024_pcon" + if system == NI: + metadata["constituency_code"]["relation"] = "official_tabulation" + return metadata + + +def _conditioning_metadata(system: str) -> dict[str, dict[str, str]]: + metadata = { + IMD_DECILE_COLUMN: { + "kind": "code", + "source": f"toy-{system}-imd-decile", + "vintage": "toy_imd", + "relation": "inferred_modal", + } + } + for column in BAND_WEIGHT_COLUMNS[system]: + metadata[column] = { + "kind": "weight", + "source": f"toy-{system}-{column}", + "basis": "toy households in the band x1e6", + } + return metadata + + +def _payload(system: str, *, rfdr_roster: bool) -> bytes: + areas = _areas(system, rfdr_roster=rfdr_roster) + households = [area.household_count() for area in areas] + rows = { + "oa_code": [area.oa_code for area in areas], + "constituency_code": [area.constituency_code for area in areas], + "region_code": [FRS_REGION_TO_REGION_CODE[area.region] for area in areas], + "lsoa_code": [area.lsoa_code for area in areas], + "msoa_code": [area.msoa_code for area in areas], + "local_authority_code": [area.local_authority_code for area in areas], + "ward_code": [area.ward_code for area in areas], + "itl3_code": [area.itl3_code for area in areas], + } + arrays = {column: np.asarray(values, dtype="U") for column, values in rows.items()} + arrays["households"] = np.asarray(households, dtype=np.int64) + arrays["population"] = np.asarray( + [2 * count + 1 for count in households], dtype=np.int64 + ) + base = assemble_uk_atomic_area_support( + system=system, arrays=arrays, column_metadata=_base_metadata(system) + ) + # The adapter sorts its rows by area; the conditioning columns follow. + order = { + code: position + for position, code in enumerate( + decode_atomic_support(base).arrays["area"].tolist() + ) + } + ranked = sorted(areas, key=lambda area: order[area.oa_code]) + band_weights = { + column: np.asarray( + [ + dict(area.bands).get(column.removeprefix("households_band_").upper(), 0) + * BAND_WEIGHT_SCALE + for area in ranked + ], + dtype=np.int64, + ) + for column in BAND_WEIGHT_COLUMNS[system] + } + return assemble_uk_conditioned_area_support( + base=base, + imd_decile=np.asarray([area.imd_decile for area in ranked], dtype="U"), + band_weights=band_weights, + column_metadata=_conditioning_metadata(system), + ) + + +@functools.cache +def _payloads(rfdr_roster: bool) -> tuple[tuple[str, bytes], ...]: + return tuple( + (system, _payload(system, rfdr_roster=rfdr_roster)) for system in SYSTEMS + ) + + +def toy_conditioned_support_payloads(*, rfdr_roster: bool = True) -> dict[str, bytes]: + """The three toy conditioned supports' canonical bytes, keyed by system. + + With ``rfdr_roster`` (the default), each Great Britain support also holds + every other output area the committed RFDR layer flags, as a filler with + no households; without it, the toy areas alone. + """ + + return dict(_payloads(bool(rfdr_roster))) + + +def toy_conditioned_supports(*, rfdr_roster: bool = True) -> dict[str, AtomicSupport]: + """The toy supports, decoded, keyed by system.""" + + return { + system: decode_atomic_support(payload) + for system, payload in toy_conditioned_support_payloads( + rfdr_roster=rfdr_roster + ).items() + } + + +def toy_household_table(households: Sequence[ToyHousehold]) -> pd.DataFrame: + """The draw's raw inputs for ``households``, typed as the spine's root + types them: an object region and band, an int64 decile (0 for none) and a + boolean band flag.""" + + return pd.DataFrame( + { + "region": pd.Series([h.region for h in households], dtype=object), + "imd_decile": np.asarray( + [h.imd_decile for h in households], dtype=np.int64 + ), + "council_tax_band": pd.Series( + [h.council_tax_band for h in households], dtype=object + ), + "council_tax_band_observed": np.asarray( + [h.council_tax_band_observed for h in households], dtype=bool + ), + } + ) + + +def toy_spine_frame( + *, clones: int = 1, households: Sequence[ToyHousehold] = TOY_HOUSEHOLDS +) -> Frame: + """``households`` as the spine holds them after its identity step. + + Synthetic FRS-shaped ids (``uk_clone_first_frames.base_id_tables``, one + to four people in one or two benefit units), the households' raw draw + inputs, invented design weights and the FRS release's survey year as the + time period; then ``clones`` geographic clones through the spine's own + expansion, each keyed by the spine's identity step. + """ + + from microcosm.build.uk_runtime.frs_release import load_uk_frs_release + from microcosm.build.uk_runtime.geographic_expand import ( + expand_uk_geographic_clones, + ) + from microcosm.build.uk_runtime.geographic_identity import ( + key_uk_households_by_clone_path, + ) + from microcosm.build.uk_runtime.national_frame import uk_national_frame + + tables = base_id_tables(len(households)) + household = pd.concat( + [ + tables.household[["household_id"]].reset_index(drop=True), + toy_household_table(households), + ], + axis=1, + ) + frame = uk_national_frame( + person=tables.person, + benunit=tables.benunit, + household=household, + household_weights=np.asarray( + [800.0 + 25.0 * position for position in range(len(households))] + ), + time_period=str(load_uk_frs_release().survey_year), + ) + expanded = expand_uk_geographic_clones(frame, clones=clones).frame + return key_uk_households_by_clone_path(expanded).frame From ba5e01683791264223f9ceb867330e1446712f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:58:18 +0000 Subject: [PATCH 066/108] Count the take-up anchors and the apprentice target within each geographic clone (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit frs_take_up's Child Benefit, Pension Credit and Universal Credit draws now fill their anchored residual within each geographic clone, and frs_person_draws walks each clone's apprentices to that clone's own target. So clone 0 of a clone-first spine takes a one-clone spine's flags, as the two stages' other draws, keyed on each row's own id, already did. This follows María's rulings of 6 and 10 Nov (decision 3, X: stages 10-13 per clone, so that clone 0 nests on the one-clone spine) and takes up O13. Stage numbers here keep the order before stages 14 and 15 moved: 10 is frs_take_up, 11 frs_person_draws, 12 frs_household_draws and 13 frs_brma. Before, one count over every clone rounded int(rate x n) once over all of them, and one walk over every clone interleaved the clones' employees and stopped once, so clone 0's flags were not one clone's. Shared, for Max's review: - stochastic_assignment.assign_binary_with_anchored_residual takes a keyword-only strata, one integer per row. Each stratum's rows take the flags they take alone: their own int(rate x n) target, the need their own anchors leave and the threshold their own non-anchored rows fill. - Without strata the function runs as before. One stratum gives the same flags bit for bit, and without an anchor strata change nothing. - Strata that do not align with the draws, and strata without an integer dtype, every column with a missing label among them, are refused in fixed sentences that name no count, label or row. The module names no country and imports nothing new. frs_take_up: - derive_frs_take_up takes clone_index and passes it as strata to the three anchored draws. A clone_index that does not align with the benefit units is refused in a fixed sentence. - The stage reads each benefit unit's clone from household_clone_index through the person memberships, with the new geographic_clone_indices, which shares its mapping with geographic_clone_zero_rows. The stage's consumes do not change, and a frame without the column passes None. frs_person_draws: - derive_is_apprentice takes person_clone_index. Each clone walks its own employees to the rate times its own weight aged 21 to 24, with the unchanged nearest-target walk. The stage reads the index through geographic_clone_indices. - The receipt keeps its keys and their formulas. The masses and the realized weight are sums over every clone, the target is the rate times the weight aged 21 to 24 over every clone, and cap_bound compares the eligible weight over every clone with that target, as stage_health's apprentice_draw gate recomputes them. The supports are still counted among clone 0's people in true source households and withheld below the floor of 3 as before. - A misaligned or non-integer index, or a person_clone_zero that disagrees with it, is refused in a fixed sentence with no count or row. E4 (tools/verify_uk_identity_stability.py) gives both recomputes each row's clone, read once from the scoped frame and carried by id onto the permuted tables, and holds stored is_apprentice flags to the walk within each clone. Its tolerance_policy text says so. Left as they are: spi_benefit_coherence's SPI Universal Credit redraw and uc_take_up still pool every clone, and the K table classes the SPI redraw's count as kept. frs_take_up's notes, frs_person_draws' rate_derivation and the take-up contract's policy line say so with the geography declarations; no declaration changes here. Text: changelog.d/931-take-up-counts-per-clone.changed.md is new. What moves. No value at one clone. Without strata, assign_binary_with_anchored_residual is unchanged byte for byte, and with every row in one stratum it gives the same flags. So a frame without household_clone_index, and a spine whose clone index is all 0, give every frs_take_up and frs_person_draws output and receipt bit for bit as before. On invented frames this tree and its parent give byte-identical outputs and receipts for both stages, with and without a one-clone index, and the same E4 receipt but for its tolerance_policy text. On the synthetic H2 fixture, through the engine slot, the legacy oracle's final frame (every table, column and dtype, the weights, the strata and the mass log), its stage records and every stage's evidence digest are identical before and after this commit, and the fixture regenerates byte-identically. Above one clone, which no build has yet run on this code, frs_take_up's would_claim_child_benefit, would_claim_pc and would_claim_uc move, and so does frs_person_draws' is_apprentice. Each clone now meets its own target. Clone 0 takes a one-clone spine's three anchored flags at any clone count. It takes that spine's is_apprentice flags wherever dividing by K scales its cumulative weights exactly, as at a power of two, and elsewhere except at a near-tie of the walk. Whatever reads these flags follows; frs_brma reads would_claim_uc through LHA_category. No receipt gains or loses a field. The identity tool's E4 recompute gives both draws each row's clone and matches them above one clone. Its tolerance_policy text now says the apprentice walk is held within each geographic clone; that is the one field that differs at one clone. Keys. The frs_spine root and 48 of the 55 stages after it hash stochastic_assignment, so every spine node re-keys, and both spine gate nodes with them. The transferred phase's own code hashes it too; the assembled phase's does not. frs_take_up and frs_person_draws are hashed only by kernels that also hash stochastic_assignment, with one exception: the full build's calibrated gates, whose own terminal code hashes frs_take_up. On an already-built spine, uk.full.gates.calibrated and the nodes after it therefore re-key without a re-solve, because the preflight's code and the solve's key leave that code out. The national gates hash both modules through the gate registry's list. No node hashes the identity tool, and no other graph key moves: of the 101 kernels the UK registries register, the root, those 48 stages and the national gates move, and the shared geography kernels, the spine and full gate kernels and every dense and national target, problem, solve, readback and export kernel keep their hashes. The 45 kernels the transport registry registers, from which NZ's graph is composed, keep theirs, and no NZ, BE, AM or US module imports an edited module. No committed file outside docs/evidence pins a hash or phase code that moves. No declaration, generated file, gates.json or schema changes, and H2 is not regenerated. Tests (invented data): - The shared keyword: - without strata, and with one stratum under three labels and integer dtypes, byte for byte against the arithmetic before it on random cases; - two hand-built strata, each stratum's flags its rows' alone, under relabelling and row permutation; - without an anchor, strata change nothing; - the refusals, with no digit in any message. - frs_take_up on the clone-first stand-in at three clones over a grid of rates: - clone 0 is the one-clone frame's and each clone its own units' alone, while one count over every clone moves clone 0; - bit for bit at one clone; - the clone reader and the refusals. - The apprentice walk: - clone 0 equal at two and four clones, and each clone its own people's alone; - an exact tie of the one-clone walk, which clone 0 keeps at 2, 4, 8 and 16 clones and breaks by at most one person at 3, 5, 6, 7 and 15; - supports and cap_bound equal at three; - bit for bit at one clone against the walk before it; - the stage at four clones, and the refusals. - test_uk_clone_first_nesting: frs_take_up and frs_person_draws through geographic_expand at 2, 3, 7 and 15 clones. - Clone 0 is the one-clone frame on every column and dtype. - Each of the four draws, counted over every clone, moves clone 0 at some K. - The K = 3 count test now also compares distinct values and the apprentice draw's flagged support and cap_bound. - E4 at three clones matches the store at two Universal Credit rates. Without the clone index it reports is_apprentice, and brma too at the rate where int(rate x units) rounds otherwise over every clone. The walk check holds within each clone. - The K table: the take-up target count is per clone, the SPI redraw's is kept, and the apprentice target is a per-clone weight. - Through the engine slot, the engine-tier module at four clones: - clone 0 through frs_brma is the one-clone frame on every column, including the three anchored flags, is_apprentice and brma; - the apprentice receipt's flagged support is one clone's. Co-Authored-By: Claude Opus 5.5 --- .../931-take-up-counts-per-clone.changed.md | 9 + .../microcosm/build/stochastic_assignment.py | 48 +++ .../build/uk_runtime/frs_person_draws.py | 117 +++++- .../microcosm/build/uk_runtime/frs_take_up.py | 89 +++- .../shared/test_stochastic_assignment.py | 252 +++++++++++ .../engine_free/uk/test_uk_apprentice_draw.py | 341 ++++++++++++++- .../uk/test_uk_clone_first_nesting.py | 390 ++++++++++++++++-- .../engine_free/uk/test_uk_frs_stochastic.py | 233 +++++++++++ .../uk/test_uk_identity_stability_receipts.py | 113 ++++- .../uk/test_uk_k_sensitive_constants.py | 85 +++- .../uk/test_acceptance_h_clone_first.py | 118 +++--- tools/verify_uk_identity_stability.py | 127 ++++-- 12 files changed, 1735 insertions(+), 187 deletions(-) create mode 100644 changelog.d/931-take-up-counts-per-clone.changed.md diff --git a/changelog.d/931-take-up-counts-per-clone.changed.md b/changelog.d/931-take-up-counts-per-clone.changed.md new file mode 100644 index 0000000000..7dec659454 --- /dev/null +++ b/changelog.d/931-take-up-counts-per-clone.changed.md @@ -0,0 +1,9 @@ +`frs_take_up`'s Child Benefit, Pension Credit and Universal Credit draws and `frs_person_draws`' apprentice walk now count within each geographic clone, so clone 0 of a clone-first spine takes a one-clone spine's flags (#931; María's rulings of 6 and 10 Nov). Before, the anchored residual took `int(rate × n)` once over every clone's benefit units, and the walk interleaved every clone's employees and stopped once, so clone 0's flags were not one clone's. The stages' other draws were already keyed on each row's own id. + +- **Shared `strata`** (for Max's review). `stochastic_assignment.assign_binary_with_anchored_residual` takes a keyword-only `strata`, one integer per row. Each stratum's rows take the flags they take alone: their own `int(rate × n)` target, the need their own anchors leave and the threshold their own non-anchored rows fill. Without `strata` the function runs as before; with every row in one stratum it gives the same flags bit for bit, and without an anchor strata change nothing. Strata that do not align with the draws, and strata without an integer dtype, every column with a missing label among them, are refused in fixed sentences that name no count, label or row. The module names no country, and only UK modules and UK tools import it. +- **frs_take_up.** `derive_frs_take_up` takes `clone_index` and passes it as `strata` to the three anchored draws. The stage reads each benefit unit's clone from `household_clone_index` through the person memberships with the new `geographic_clone_indices`, which shares its mapping with `geographic_clone_zero_rows`, so its consumes do not change. A frame without the column passes None. +- **frs_person_draws.** `derive_is_apprentice` takes `person_clone_index`, as do `derive_frs_person_draws` and `derive_frs_person_draws_with_receipt`, and each clone walks its own employees to the rate times its own weight aged 21 to 24, with the unchanged nearest-target walk. The stage reads the index through `geographic_clone_indices`. The receipt keeps its keys and their formulas: the masses and the realized weight are sums over every clone, `target` is the rate times the weight aged 21 to 24 over every clone, and `cap_bound` compares the eligible weight over every clone with that target, as the `apprentice_draw` stage-health check recomputes them. The supports are still counted among clone 0's people in true source households, and withheld below the floor of three as before. A misaligned or non-integer index, or a `person_clone_zero` that disagrees with it, is refused in a fixed sentence that names no count or row. +- **E4** (`tools/verify_uk_identity_stability.py`) gives both recomputes each row's clone, read once from the scoped frame and carried by id onto the permuted tables, and holds stored `is_apprentice` flags to the walk within each clone. Its `tolerance_policy` text says so. +- **What moves.** At one clone, no value: both stages' outputs and receipts are as before, bit for bit, with or without a one-clone index, and E4's receipt differs only in its `tolerance_policy` text. On the synthetic H2 fixture the legacy oracle's final frame, its stage records and every stage's evidence digest are identical before and after, and the fixture regenerates byte-identically. Above one clone, which no build has yet run on this code, the three anchored flags and `is_apprentice` move, each clone meeting its own target. Clone 0's anchored flags are one clone's exactly. Its `is_apprentice` flags are one clone's at any power-of-two clone count, and elsewhere except at a near-tie of the walk. Whatever reads these flags follows: `frs_brma` reads `would_claim_uc` through `LHA_category`. `spi_benefit_coherence`'s Universal Credit redraw and `uc_take_up` still count over every clone. +- **Keys.** The frs_spine root and 48 of the 55 stages after it hash `stochastic_assignment`, so every spine node re-keys, and both spine gate nodes with them. Of the gate phases' own codes, the transferred spine phase's hashes `stochastic_assignment` and moves; the assembled phase's does not. On a bound spine the national gates re-key, and so do the full build's calibrated gates and the nodes after them, through the terminal phase's code, which hashes `frs_take_up`; they do so without a re-solve, since the preflight's code and the solve's key leave that code out. No node hashes the identity tool. The 45 kernels the transport registry registers, from which NZ's graph is composed, keep their hashes, and no NZ, BE, AM or US module imports an edited file. No declaration, generated file, `gates.json` or schema changes: `frs_take_up`'s notes, `frs_person_draws`' `rate_derivation` and the take-up contract's policy line follow with the geography declarations. +- **Tests** (invented data). The shared keyword against its arithmetic before strata, without strata and with one stratum in three integer dtypes; two hand-built strata, under relabelling and row permutation; strata without an anchor; and the refusals. `frs_take_up` on the clone-first stand-in at three clones over a grid of rates, and bit for bit at one clone. The walk at two and four clones, its supports and cap bound at three, an exact tie of the walk that clone 0 keeps at every power-of-two clone count up to 16 and breaks by at most one person elsewhere, the stage at four clones, and bit for bit at one clone. Both stages through `geographic_expand` at 2, 3, 7 and 15 clones, where counting over every clone instead moves clone 0 for each of the four draws. E4 at three clones at two Universal Credit rates, and the walk check within each clone. The engine-tier test at four clones, where clone 0 through `frs_brma` is now the one-clone frame on every column. diff --git a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py index a4894ba5bc..61224eff57 100644 --- a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py +++ b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py @@ -245,6 +245,8 @@ def assign_binary_with_anchored_residual( rate: float, anchor: Sequence[bool] | np.ndarray | None = None, population: Sequence[bool] | np.ndarray | None = None, + *, + strata: Sequence[int] | np.ndarray | None = None, ) -> np.ndarray: """Assign a flag while forcing reported-recipient anchors to true. @@ -255,6 +257,22 @@ def assign_binary_with_anchored_residual( rows and rows outside it are never drawn. Anchored overshoot is accepted, an anchor outside the population stays true (reported receipt is a fact), and the residual fills only non-anchored rows inside the population. + + ``strata``, one integer per row, takes the count within each stratum + rather than over every row: each stratum runs the arithmetic above on its + own rows alone, with its own target, the need its own anchors leave and + the threshold its own non-anchored rows fill, so their order does not + matter. A stratum's flags are then the flags its rows take on their own, + whatever the other strata hold, and with every row in one stratum they + are the flags without strata. A caller whose strata are copies of one + population, as geographic clones are (microcosm#931), counts each copy as + the population alone is counted, where one count over every copy rounds + ``int(rate * n_units)`` once over all of them. Without an anchor there is + no count, so strata change nothing, though they are still checked: they + must align with ``draws`` and have an integer dtype, which holds no + missing value. Unlike :func:`weight_balanced_uniforms`' strata, the + labels must be integers: a label a merge left missing would match no + stratum, and its rows would keep their anchors alone. """ draws = np.asarray(draws, dtype=np.float64) @@ -265,11 +283,21 @@ def assign_binary_with_anchored_residual( population = np.asarray(population, dtype=bool) if population.shape != draws.shape: raise ValueError("population and draws must align") + if strata is not None: + strata = _stratum_labels(strata, draws.shape) if anchor is None: return population & (draws < rate) anchor = np.asarray(anchor, dtype=bool) if anchor.shape != draws.shape: raise ValueError("anchor and draws must align") + if strata is not None: + result = anchor.copy() + for stratum in np.unique(strata): + rows = strata == stratum + result[rows] = assign_binary_with_anchored_residual( + draws[rows], rate, anchor[rows], population[rows] + ) + return result result = anchor.copy() target = int(rate * int(population.sum())) remaining_needed = max(0, target - int((anchor & population).sum())) @@ -330,3 +358,23 @@ def _validate_rate(rate: Any) -> float: if not 0.0 <= value <= 1.0: raise ValueError(f"rate must be in [0, 1], got {value}.") return value + + +def _stratum_labels( + strata: Sequence[int] | np.ndarray, shape: tuple[int, ...] +) -> np.ndarray: + """``strata`` as one integer label per row, or a refusal. + + The labels must align with the draws and have an integer dtype, which + holds no missing value: a float holding whole numbers is refused, as + :func:`integer_record_keys` refuses float ids, and so is a float, object + or nullable column holding a missing value. The refusals name no count, + label or row. + """ + + labels = np.asarray(strata) + if labels.shape != shape: + raise ValueError("strata and draws must align") + if labels.dtype.kind not in "iu": + raise TypeError("strata must be integers, one per row, none missing") + return labels diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py index b9a4ed6516..ec5609a37b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py @@ -50,6 +50,7 @@ ) from microcosm.build.uk_runtime.frs_employment import FRS_EMPSTATI_EMPLOYMENT_STATUS from microcosm.build.uk_runtime.frs_take_up import ( + geographic_clone_indices, geographic_clone_zero_rows, take_up_signal_outputs, uk_take_up_signal_receipt, @@ -188,6 +189,9 @@ def add_frs_person_draws_with_receipt( frame, "person", stage="frs_person_draws", before_copies=True ), person_clone_zero=geographic_clone_zero_rows(frame, "person"), + # Each person's household's clone, which the apprentice walk meets its + # target within; None on a frame without household_clone_index. + person_clone_index=geographic_clone_indices(frame, "person"), ) for column in FRS_PERSON_DRAW_OUTPUT_COLUMNS: person[column] = derived[column].to_numpy() @@ -212,13 +216,17 @@ def derive_frs_person_draws( relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, person_clone_zero: np.ndarray | None = None, + person_clone_index: np.ndarray | None = None, ) -> pd.DataFrame: """Return the stage's person draws. ``person_weights`` are each person's household weight, which set - ``is_apprentice``'s target; ``person_sources`` are each person's true - source household, which its receipt counts support by, among geographic - clone 0's people (``person_clone_zero``; everyone when not given). + ``is_apprentice``'s target, met within each geographic clone of + ``person_clone_index`` (every person in one when not given); + ``person_sources`` are each person's true source household, which its + receipt counts support by, among geographic clone 0's people + (``person_clone_zero``; where it is not given, clone 0 of + ``person_clone_index``, else everyone). """ return derive_frs_person_draws_with_receipt( @@ -228,6 +236,7 @@ def derive_frs_person_draws( relief_schedule=relief_schedule, person_sources=person_sources, person_clone_zero=person_clone_zero, + person_clone_index=person_clone_index, )[0] @@ -239,6 +248,7 @@ def derive_frs_person_draws_with_receipt( relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, person_clone_zero: np.ndarray | None = None, + person_clone_index: np.ndarray | None = None, ) -> tuple[pd.DataFrame, Mapping[str, object]]: """The stage's person draws and the receipt of its ``is_apprentice`` draw.""" @@ -273,6 +283,7 @@ def derive_frs_person_draws_with_receipt( relief_schedule=relief_schedule, person_sources=person_sources, person_clone_zero=person_clone_zero, + person_clone_index=person_clone_index, ) values[APPRENTICE_OUTPUT] = apprentices.flags return values, {"build_year": contract.build_year, **apprentices.receipt} @@ -294,6 +305,7 @@ def derive_is_apprentice( relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, person_clone_zero: np.ndarray | None = None, + person_clone_index: np.ndarray | None = None, ) -> UKApprenticeDraw: """Draw ``is_apprentice`` for employees aged 21 to 24 outside full-time education. @@ -306,6 +318,23 @@ def derive_is_apprentice( eligible weight falls short of the target every eligible employee is flagged and the receipt says the draw was cap-bound. + The target is met within each geographic clone (microcosm#931). + ``person_clone_index`` gives each person's clone (everyone is in one + clone when it is not given), and each clone walks its own employees to + the rate times its own weight aged 21 to 24. Clone 0 holds a one-clone + spine's people at their weights over K, so it takes that spine's flags + at any power-of-two K, which scales its cumulative weights exactly, and + elsewhere except at a near-tie: where the target lies within rounding of + the midpoint between two consecutive prefixes of the walk, or of the + eligible weight, at which the walk flags everyone. One walk over every + clone would interleave the clones' employees and stop once, so clone 0's + flags would not be one clone's. The receipt's masses and realized weight + are sums over the clones. Its target is the rate times the weight aged + 21 to 24 over every clone, and the draw is cap-bound where the eligible + weight over every clone is no more than that target: the figures + stage_health's apprentice_draw gate recomputes from the receipt, and at + one clone the walk's own, bit for bit. + Given a ``relief_schedule`` (the engine's Class 1 rate and its secondary and apprentice upper secondary thresholds, annual), the receipt also records the relief the flags imply: the rate on each flagged employee's @@ -317,11 +346,12 @@ def derive_is_apprentice( without the frame's lineage passes ``own_source_households`` itself, so no table is counted by its rows unawares (microcosm#931). The supports it prints are counted among geographic clone 0's people, - ``person_clone_zero`` (everyone when not given): each clone flags its own - employees, so the flagged sources over every clone would grow with the - clone count. The masses are taken over every clone, so each is withheld - where one or two source households carry it in clone 0's people or over - every clone: clone 0 can flag none of the employees one or two + ``person_clone_zero`` (where it is not given, the people + ``person_clone_index`` puts in clone 0, else everyone): each clone flags + its own employees, so the flagged sources over every clone would grow + with the clone count. The masses are taken over every clone, so each is + withheld where one or two source households carry it in clone 0's people + or over every clone: clone 0 can flag none of the employees one or two households hold in the other clones. """ @@ -344,6 +374,7 @@ def derive_is_apprentice( ) if weights.shape != (len(person),): raise ValueError("person_weights must hold one weight per person.") + clones = _clone_rows(person_clone_index, len(person)) age = pd.to_numeric(person["age"], errors="coerce").to_numpy(dtype=float) in_ages = (age >= APPRENTICE_MIN_AGE) & (age < APPRENTICE_MAX_AGE_EXCLUSIVE) employee = ( @@ -359,22 +390,24 @@ def derive_is_apprentice( draws = _draws(person["person_id"].to_numpy(), APPRENTICE_OUTPUT) population_mass = float(weights[in_ages].sum()) eligible_mass = float(weights[eligible].sum()) + flags = np.zeros(len(person), dtype=bool) + for rows in clones: + clone_eligible = eligible & rows + threshold = _nearest_target_threshold( + float(rate) * float(weights[in_ages & rows].sum()), + draws=draws[clone_eligible], + weights=weights[clone_eligible], + ) + flags |= clone_eligible & (draws < threshold) + # The receipt's target and cap bound are taken over every clone, as + # stage_health's apprentice_draw gate recomputes them from the receipt's + # masses; at one clone they are the walk's own, bit for bit. target = float(rate) * population_mass - threshold = _nearest_target_threshold( - target, draws=draws[eligible], weights=weights[eligible] - ) - flags = eligible & (draws < threshold) realized = float(weights[flags].sum()) sources = np.asarray(person_sources) if sources.shape != (len(person),): raise ValueError("person_sources must hold one source per person.") - clone_zero = ( - np.ones(len(person), dtype=bool) - if person_clone_zero is None - else np.asarray(person_clone_zero, dtype=bool) - ) - if clone_zero.shape != (len(person),): - raise ValueError("person_clone_zero must hold one flag per person.") + clone_zero = _clone_zero_people(person_clone_zero, person_clone_index, len(person)) population_support = distinct_count(sources, in_ages & clone_zero, weights) eligible_support = distinct_count(sources, eligible & clone_zero, weights) flagged_support = distinct_count(sources, flags & clone_zero, weights) @@ -445,6 +478,52 @@ def _thin(support: int) -> bool: return 0 < support < UK_DISCLOSURE_FLOOR +def _clone_rows(clone_index: np.ndarray | None, rows: int) -> tuple[np.ndarray, ...]: + """Each geographic clone's rows of ``rows`` people, in sorted clone order. + + A ``clone_index`` that is None or holds one clone gives one mask of every + row, so a one-clone frame's walk runs on its rows as a frame without the + index does. The index must hold one integer per person; the refusals name + no count, clone or row. + """ + + if clone_index is None: + return (np.ones(rows, dtype=bool),) + labels = np.asarray(clone_index) + if labels.shape != (rows,): + raise ValueError("person_clone_index must hold one clone per person.") + if labels.dtype.kind not in "iu": + raise TypeError("person_clone_index must hold an integer clone per person.") + clones = np.unique(labels) + if clones.size <= 1: + return (np.ones(rows, dtype=bool),) + return tuple(labels == clone for clone in clones) + + +def _clone_zero_people( + clone_zero: np.ndarray | None, clone_index: np.ndarray | None, rows: int +) -> np.ndarray: + """The people the receipt counts support among: ``clone_zero``, or where + it is not given the people ``clone_index`` puts in clone 0, or everyone. + Both given, they must agree; the refusals name no count or row.""" + + if clone_zero is None: + if clone_index is None: + return np.ones(rows, dtype=bool) + return np.asarray(clone_index) == 0 + flags = np.asarray(clone_zero, dtype=bool) + if flags.shape != (rows,): + raise ValueError("person_clone_zero must hold one flag per person.") + if clone_index is not None and not np.array_equal( + flags, np.asarray(clone_index) == 0 + ): + raise ValueError( + "person_clone_zero must flag exactly the people whose " + "person_clone_index is zero." + ) + return flags + + def _nearest_target_threshold( need: float, *, draws: np.ndarray, weights: np.ndarray ) -> float: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py index 4ea3c505ef..31a5d648c4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py @@ -406,7 +406,13 @@ def add_frs_take_up( anchors = aggregate_person_reported_to_benunit(person, benunit) uc_age_eligible = uc_age_eligible_benunits(person, benunit, population_policy) derived = derive_frs_take_up( - benunit, anchors=anchors, contract=contract, uc_age_eligible=uc_age_eligible + benunit, + anchors=anchors, + contract=contract, + uc_age_eligible=uc_age_eligible, + # Each unit's household's clone, read through the person table's + # memberships; None on a frame without household_clone_index. + clone_index=geographic_clone_indices(frame, "benunit"), ) for column in FRS_TAKE_UP_OUTPUT_COLUMNS: benunit[column] = derived[column].to_numpy() @@ -480,16 +486,34 @@ def derive_frs_take_up( anchors: pd.DataFrame, contract: UKTakeUpContract, uc_age_eligible: np.ndarray, + clone_index: np.ndarray | None = None, ) -> pd.DataFrame: + """The stage's benefit-unit outputs, each drawn on ``benunit_id`` under + the output's own salt. + + The Child Benefit, Pension Credit and Universal Credit flags fill their + anchored residuals within each geographic clone (microcosm#931): + ``clone_index``, aligned with ``benunit``, gives each unit's clone and is + the anchored draws' ``strata``. Clone 0's units have a one-clone spine's + ids, anchors and population, so they take its flags exactly at any clone + count; one count over every clone would round the target once over all + of them and move clone 0's threshold. ``None``, for a frame without the + clone index, counts over every unit, as clone indices that are all 0 do. + """ + ids = benunit["benunit_id"].to_numpy() values = pd.DataFrame(index=benunit.index) population = np.asarray(uc_age_eligible, dtype=bool) if population.shape != ids.shape: raise ValueError("uc_age_eligible must align with the benefit-unit table") + strata = None if clone_index is None else np.asarray(clone_index) + if strata is not None and strata.shape != ids.shape: + raise ValueError("clone_index must align with the benefit-unit table") values["would_claim_child_benefit"] = assign_binary_with_anchored_residual( _draws(ids, "would_claim_child_benefit"), contract.rate("child_benefit"), anchors["child_benefit_reported_anchor"].to_numpy(dtype=bool), + strata=strata, ) values["child_benefit_opts_out"] = assign_binary_from_rate( _draws(ids, "child_benefit_opts_out"), @@ -499,12 +523,14 @@ def derive_frs_take_up( _draws(ids, "would_claim_pc"), contract.rate("pension_credit"), anchors["pension_credit_reported_anchor"].to_numpy(dtype=bool), + strata=strata, ) values["would_claim_uc"] = assign_binary_with_anchored_residual( _draws(ids, "would_claim_uc"), contract.rate("universal_credit"), anchors["universal_credit_reported_anchor"].to_numpy(dtype=bool), population=population, + strata=strata, ) childcare = derive_childcare_take_up(ids, contract=contract) for output, _ in UK_CHILDCARE_TAKE_UP_FLAGS: @@ -744,36 +770,71 @@ def geographic_clone_zero_rows(frame: Frame, entity: str) -> np.ndarray: """ household = frame.table("household") - table = frame.table(entity) column = ladder_clone_index_column("household") if column not in household.columns: - return np.ones(len(table), dtype=bool) - zero = pd.Series( - household[column].to_numpy() == 0, - index=household[UK_NATIONAL_SCHEMA.id_column("household")].to_numpy(), + return np.ones(len(frame.table(entity)), dtype=bool) + return np.asarray( + _household_values_on_rows(frame, entity, household[column].to_numpy() == 0), + dtype=bool, + ) + + +def geographic_clone_indices(frame: Frame, entity: str) -> np.ndarray | None: + """Each ``entity`` row's geographic clone, or ``None`` (microcosm#931). + + A row's clone is its household's ``household_clone_index``, as + :func:`geographic_clone_zero_rows` reads it: a person's household's, or + that of the household a benefit unit's members share, through the person + table's memberships, which a stage's frame always carries. + ``frs_take_up`` and ``frs_person_draws`` consume the household column + alone, so their draws count within each clone without reading another + table's index. A frame without the column (a unit fixture) is one clone + and gives ``None``. + """ + + household = frame.table("household") + column = ladder_clone_index_column("household") + if column not in household.columns: + return None + return _household_values_on_rows(frame, entity, household[column].to_numpy()) + + +def _household_values_on_rows( + frame: Frame, entity: str, values: np.ndarray +) -> np.ndarray: + """``values``, one per household row, on each ``entity`` row: a person's + household's, or that of the household a benefit unit's members share. + The refusals name no id, count or value.""" + + household = frame.table("household") + by_household = pd.Series( + values, index=household[UK_NATIONAL_SCHEMA.id_column("household")].to_numpy() ) if entity == "household": - return zero.to_numpy(dtype=bool) + return by_household.to_numpy() person = frame.table(UK_NATIONAL_SCHEMA.person_entity) - person_zero = person[UK_NATIONAL_SCHEMA.membership_column("household")].map(zero) - if person_zero.isna().any(): + on_person = person[UK_NATIONAL_SCHEMA.membership_column("household")].map( + by_household + ) + if on_person.isna().any(): raise ValueError( "Every person needs a household on the frame to read its geographic clone." ) if entity == UK_NATIONAL_SCHEMA.person_entity: - return person_zero.to_numpy(dtype=bool) + return on_person.to_numpy() by_unit = ( pd.Series( - person_zero.to_numpy(dtype=bool), + on_person.to_numpy(), index=person[UK_NATIONAL_SCHEMA.membership_column(entity)].to_numpy(), ) .groupby(level=0) .first() ) - unit_zero = table[UK_NATIONAL_SCHEMA.id_column(entity)].map(by_unit) - if unit_zero.isna().any(): + table = frame.table(entity) + on_unit = table[UK_NATIONAL_SCHEMA.id_column(entity)].map(by_unit) + if on_unit.isna().any(): raise ValueError(f"A {entity} without members has no geographic clone to read.") - return unit_zero.to_numpy(dtype=bool) + return on_unit.to_numpy() @dataclass(frozen=True) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py index 7ecb814bcb..aecbce1c36 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_stochastic_assignment.py @@ -3,6 +3,7 @@ import math import numpy as np +import pandas as pd import pytest from microcosm.build.stochastic_assignment import ( @@ -193,6 +194,257 @@ def test_anchored_residual_population_without_anchor_and_shape_refusal() -> None ) +# --- the anchored residual within strata (microcosm#931) --------------------- + + +def _anchored_residual_without_strata(draws, rate, anchor=None, population=None): + """The anchored residual's arithmetic as it stood before ``strata`` was + added, copied unchanged (its refusals aside), as the oracle that the + calls without strata, and with every row in one stratum, are held to + byte for byte.""" + + draws = np.asarray(draws, dtype=np.float64) + if population is None: + population = np.ones(draws.shape, dtype=bool) + else: + population = np.asarray(population, dtype=bool) + if anchor is None: + return population & (draws < rate) + anchor = np.asarray(anchor, dtype=bool) + result = anchor.copy() + target = int(rate * int(population.sum())) + remaining_needed = max(0, target - int((anchor & population).sum())) + non_anchored = ~anchor & population + if remaining_needed == 0 or not non_anchored.any(): + return result + adjusted = remaining_needed / int(non_anchored.sum()) + result |= non_anchored & (draws < adjusted) + return result + + +def _random_anchored_cases(seed: int, trials: int): + """Invented draws, rates, anchors and populations, with or without each. + + Draws sometimes repeat or sit on the rate; anchors and populations run + from none to every row, so the residual fills, overshoots, has no row left + to fill and meets no anchor.""" + + rng = np.random.default_rng(seed) + densities = (0.0, 0.05, 0.3, 0.9, 1.0) + for trial in range(trials): + n = int(rng.integers(0, 120)) + draws = rng.random(n) + if trial % 4 == 0 and n: + draws = np.round(draws, 1) + rate = float(rng.choice([0.0, 1.0, 0.5, *rng.random(3)])) + if trial % 5 == 0 and n: + draws[int(rng.integers(0, n))] = rate + anchor = ( + None + if trial % 6 == 0 + else rng.random(n) < densities[trial % len(densities)] + ) + population = ( + None if trial % 3 == 0 else rng.random(n) < float(rng.choice(densities)) + ) + yield draws, rate, anchor, population + + +def test_without_strata_or_in_one_stratum_the_flags_are_unchanged() -> None: + """``strata=None``, the default, and every row in one stratum, under three + labels and integer dtypes, give the flags the arithmetic gave before + strata were added, byte for byte.""" + + branches = {"no anchor": 0, "filled": 0, "nothing to fill": 0} + for draws, rate, anchor, population in _random_anchored_cases(11, 600): + expected = _anchored_residual_without_strata(draws, rate, anchor, population) + assert expected.dtype == np.bool_ + n = draws.size + results = [ + assign_binary_with_anchored_residual(draws, rate, anchor, population), + assign_binary_with_anchored_residual( + draws, rate, anchor, population, strata=None + ), + assign_binary_with_anchored_residual( + draws, rate, anchor, population, strata=np.zeros(n, dtype=np.int64) + ), + assign_binary_with_anchored_residual( + draws, rate, anchor, population, strata=np.full(n, -7, dtype=np.int8) + ), + assign_binary_with_anchored_residual( + draws, rate, anchor, population, strata=np.full(n, 3, dtype=np.uint16) + ), + ] + for result in results: + assert result.dtype == expected.dtype + assert result.shape == expected.shape + assert result.tobytes() == expected.tobytes() + if anchor is None: + branches["no anchor"] += 1 + elif (expected & ~anchor).any(): + branches["filled"] += 1 + else: + branches["nothing to fill"] += 1 + assert min(branches.values()) > 50, branches + + +def _two_strata() -> pd.DataFrame: + """Two invented strata, interleaved, at a rate of one half. + + Stratum -2: eight reachable rows, one anchored, so its target is + int(0.5 x 8) = 4 and its residual fills 3 of its 7 other rows, at the + threshold 3 / 7, which the draws (2i + 1) / 14 cross after their third. + Stratum 5: six rows, four reachable, three of those anchored, and an + anchor outside the reach: its target int(0.5 x 4) = 2 is already + exceeded, so it fills nothing. One count over both strata would aim at + int(0.5 x 12) = 6, need 2 of its 8 unanchored reachable rows, at the + threshold 2 / 8, and so take three: stratum 5's one (drawn at 0.1) and + two of stratum -2's.""" + + rows = [ + # (stratum, draw, anchor, reachable, with strata, without strata) + (5, 0.10, False, True, False, True), + (-2, 9 / 14, False, True, False, False), + (-2, 0.90, True, True, True, True), + (5, 0.95, True, True, True, True), + (-2, 1 / 14, False, True, True, True), + (5, 0.40, True, True, True, True), + (-2, 13 / 14, False, True, False, False), + (-2, 5 / 14, False, True, True, False), + (5, 0.01, False, False, False, False), + (-2, 3 / 14, False, True, True, True), + (5, 0.70, True, False, True, True), + (-2, 11 / 14, False, True, False, False), + (5, 0.20, True, True, True, True), + (-2, 7 / 14, False, True, False, False), + ] + return pd.DataFrame( + rows, + columns=["stratum", "draw", "anchor", "reachable", "strata", "pooled"], + ) + + +def test_strata_count_each_stratums_target_on_its_own_rows() -> None: + """Each stratum meets its own target on its own rows (:func:`_two_strata`), + where one count over both meets one target over every row.""" + + table = _two_strata() + draws = table["draw"].to_numpy() + anchor = table["anchor"].to_numpy() + population = table["reachable"].to_numpy() + strata = table["stratum"].to_numpy(dtype=np.int64) + + result = assign_binary_with_anchored_residual( + draws, 0.5, anchor, population, strata=strata + ) + + assert result.tolist() == table["strata"].tolist() + pooled = assign_binary_with_anchored_residual(draws, 0.5, anchor, population) + assert pooled.tolist() == table["pooled"].tolist() + # Each stratum's flags are those its rows take alone. + for label in (-2, 5): + rows = strata == label + np.testing.assert_array_equal( + result[rows], + assign_binary_with_anchored_residual( + draws[rows], 0.5, anchor[rows], population[rows] + ), + ) + # The labels only name the strata, and the rows' order does not matter. + renamed = np.where(strata == 5, 0, 1).astype(np.int32) + np.testing.assert_array_equal( + assign_binary_with_anchored_residual( + draws, 0.5, anchor, population, strata=renamed + ), + result, + ) + order = np.random.default_rng(2).permutation(len(table)) + np.testing.assert_array_equal( + assign_binary_with_anchored_residual( + draws[order], + 0.5, + anchor[order], + population[order], + strata=strata[order], + ), + result[order], + ) + + +def test_a_stratum_takes_its_own_flags_whatever_the_others_hold() -> None: + """Stratum 0 takes the flags its rows take alone at every rate, beside + strata of other draws over the same anchors and reach, as geographic + clone 0 does beside the other clones (microcosm#931). One count over every + stratum moves them at some rates, so the strata are not idle.""" + + rng = np.random.default_rng(17) + n, copies = 40, 3 + draws = rng.random(n) + anchor = rng.random(n) < 0.15 + population = rng.random(n) < 0.8 + stacked_draws = np.concatenate([draws, rng.random(n * (copies - 1))]) + stacked_anchor = np.tile(anchor, copies) + stacked_population = np.tile(population, copies) + strata = np.repeat(np.arange(copies, dtype=np.int64), n) + pooled_moves = 0 + for rate in np.linspace(0.0, 1.0, 81): + alone = assign_binary_with_anchored_residual(draws, rate, anchor, population) + stratified = assign_binary_with_anchored_residual( + stacked_draws, rate, stacked_anchor, stacked_population, strata=strata + ) + np.testing.assert_array_equal(stratified[:n], alone) + pooled = assign_binary_with_anchored_residual( + stacked_draws, rate, stacked_anchor, stacked_population + ) + pooled_moves += not np.array_equal(pooled[:n], alone) + assert pooled_moves > 0 + + +def test_without_an_anchor_strata_change_nothing() -> None: + rng = np.random.default_rng(8) + draws = rng.random(50) + population = rng.random(50) < 0.6 + strata = rng.integers(0, 4, 50) + + for rate in (0.0, 0.3, 1.0): + expected = population & (draws < rate) + np.testing.assert_array_equal( + assign_binary_with_anchored_residual( + draws, rate, population=population, strata=strata + ), + expected, + ) + np.testing.assert_array_equal( + assign_binary_with_anchored_residual(draws, rate, strata=strata), + draws < rate, + ) + + +@pytest.mark.parametrize("with_anchor", [True, False]) +def test_strata_are_refused_misaligned_non_integer_or_missing(with_anchor) -> None: + """Strata are checked whether or not an anchor is given, and every refusal + names no count, label or row. An integer dtype holds no missing value, so + a missing label is refused with the other dtypes that are not integers.""" + + draws = np.linspace(0.0, 0.9, 4) + anchor = np.array([True, False, False, False]) if with_anchor else None + refused = ( + (np.zeros(3, dtype=np.int64), ValueError, "strata and draws must align"), + (np.zeros((2, 2), dtype=np.int64), ValueError, "strata and draws must align"), + (np.zeros(4), TypeError, "strata must be integers"), + (np.zeros(4, dtype=bool), TypeError, "strata must be integers"), + (np.array([0, 1, 0, 1], dtype=object), TypeError, "strata must be integers"), + (np.array(["a", "b", "a", "b"]), TypeError, "strata must be integers"), + (np.array([0.0, np.nan, 1.0, 1.0]), TypeError, "none missing"), + (pd.array([0, None, 1, 1], dtype="Int64"), TypeError, "none missing"), + (np.array([0, None, 1, 1], dtype=object), TypeError, "none missing"), + ) + for strata, error, message in refused: + with pytest.raises(error, match=message) as caught: + assign_binary_with_anchored_residual(draws, 0.5, anchor, strata=strata) + assert not any(character.isdigit() for character in str(caught.value)) + + def test_integer_record_keys_pass_unique_integer_ids_through() -> None: ids = np.array([3, 1, 2], dtype="int64") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_apprentice_draw.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_apprentice_draw.py index 5697abfff7..35f4c8980c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_apprentice_draw.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_apprentice_draw.py @@ -18,7 +18,16 @@ derive_is_apprentice, ) from microcosm.build.uk_runtime.national_frame import uk_national_frame -from microcosm.build.uk_runtime.receipt_counts import own_source_households +from microcosm.build.uk_runtime.receipt_counts import ( + UK_WITHHELD, + distinct_count, + own_source_households, + support_band, +) +from test_support.microcosm_build.uk_clone_first_frames import ( + GEOGRAPHIC_RADIX, + clone_first_frame, +) from test_support.microcosm_build.uk_frs_stochastic import _Contract @@ -325,6 +334,336 @@ def test_receipt_withholds_what_other_clones_flag_on_one_or_two_households() -> assert draw.receipt["population_mass"] == float(len(person)) +# --- the walk within each geographic clone (microcosm#931) ----------------------- + +#: Every rate per head from 0 to 1 in steps of 0.05. +_CLONE_RATES = tuple(step / 20 for step in range(21)) +#: The receipt's fields that do not move with K: its supports, counted in +#: clone 0's people, and whether the draw is cap-bound, which compares the +#: eligible weight and the target over every clone, one clone's to rounding. +_K_INVARIANT_FIELDS = ( + "population_support", + "eligible_support", + "flagged_support", + "cap_bound", +) + + +def _clone_base(households: int = 90) -> tuple[pd.DataFrame, np.ndarray]: + """Invented people aged 18 to 30 of the four statuses in households of + one to three, each with its household's weight.""" + + rows = [ + ( + household * 1000 + member, + household, + 18 + (7 * household + 3 * member) % 13, + _STATUSES[(household + member) % 4], + ) + for household in range(1, households + 1) + for member in range(1, 2 + household % 3) + ] + person = pd.DataFrame( + rows, columns=["person_id", "person_household_id", "age", "employment_status"] + ) + household_weights = np.random.default_rng(931).uniform(100.0, 3000.0, households) + return person, household_weights[person["person_household_id"].to_numpy() - 1] + + +def _as_clones( + person: pd.DataFrame, weights: np.ndarray, clones: int +) -> tuple[pd.DataFrame, np.ndarray, np.ndarray, np.ndarray]: + """``clones`` geographic clones of ``person``, clone-major, as the + clone-first spine makes them: clone k's ids plus k x the geographic + radix, each at ``w / clones`` of its weight. Returns the people, their + weights, their clone index and their true source households.""" + + people = pd.concat( + [ + person.assign( + person_id=person["person_id"] + k * GEOGRAPHIC_RADIX, + person_household_id=person["person_household_id"] + + k * GEOGRAPHIC_RADIX, + ) + for k in range(clones) + ], + ignore_index=True, + ) + return ( + people, + np.tile(weights / clones, clones), + np.repeat(np.arange(clones, dtype=np.int64), len(person)), + np.tile(person["person_household_id"].to_numpy(), clones), + ) + + +@pytest.mark.parametrize("clones", [2, 4]) +def test_clone_zero_walks_as_one_clone_at_two_and_four_clones(clones: int) -> None: + """Each clone walks its own employees to the rate times its own weight + aged 21 to 24. Clone 0's people are one clone's at their weights over K, + which a power of two divides exactly, so at every rate clone 0 takes the + one-clone flags and every clone the flags its own people take alone. The + supports, counted in clone 0's people, and whether the draw is cap-bound + are one clone's; the masses are summed over the clones. One walk over + every clone moves clone 0's flags at some rates (microcosm#931).""" + + base, weights = _clone_base() + sources = base["person_household_id"].to_numpy() + person, clone_weights, index, clone_sources = _as_clones(base, weights, clones) + pooled_moves = 0 + for rate in _CLONE_RATES: + one = derive_is_apprentice( + base, rate=rate, person_weights=weights, person_sources=sources + ) + many = derive_is_apprentice( + person, + rate=rate, + person_weights=clone_weights, + person_sources=clone_sources, + person_clone_zero=index == 0, + person_clone_index=index, + ) + assert many.flags[index == 0].tolist() == one.flags.tolist(), rate + for clone in range(clones): + rows = index == clone + alone = derive_is_apprentice( + person.loc[rows].reset_index(drop=True), + rate=rate, + person_weights=clone_weights[rows], + person_sources=clone_sources[rows], + ) + assert many.flags[rows].tolist() == alone.flags.tolist(), (rate, clone) + for key in _K_INVARIANT_FIELDS: + assert many.receipt[key] == one.receipt[key], (rate, key) + for key in ("population_mass", "eligible_mass", "target"): + assert many.receipt[key] == pytest.approx(one.receipt[key], rel=1e-12) + pooled = derive_is_apprentice( + person, + rate=rate, + person_weights=clone_weights, + person_sources=clone_sources, + person_clone_zero=index == 0, + ) + pooled_moves += pooled.flags[index == 0].tolist() != one.flags.tolist() + assert pooled_moves + + +def test_an_exact_tie_nests_clone_zero_at_a_power_of_two_only() -> None: + """Four employees of weight 1 at a rate of 0.625 put the one-clone walk's + target, 2.5, midway between flagging two of them and three; the walk + flags three. Dividing every weight by a power of two scales the tie + exactly, so clone 0 keeps those flags at K = 2, 4, 8 and 16, which the + engine tier's exact K = 4 class rests on. At K = 3, 5, 6, 7 and 15, + w / K rounds and can break the tie by one person, the near-tie + derive_is_apprentice names (microcosm#931).""" + + base = _small([22] * 4, ["FT_EMPLOYED"] * 4) + weights = np.ones(len(base)) + sources = base["person_household_id"].to_numpy() + one = derive_is_apprentice( + base, rate=0.625, person_weights=weights, person_sources=sources + ) + assert int(one.flags.sum()) == 3 + moved = {} + for clones in (2, 3, 4, 5, 6, 7, 8, 15, 16): + person, clone_weights, index, clone_sources = _as_clones(base, weights, clones) + many = derive_is_apprentice( + person, + rate=0.625, + person_weights=clone_weights, + person_sources=clone_sources, + person_clone_index=index, + ) + moved[clones] = int((many.flags[index == 0] != one.flags).sum()) + assert all(moved[k] == 0 for k in (2, 4, 8, 16)), moved + assert all(moved[k] <= 1 for k in (3, 5, 6, 7, 15)), moved + assert any(moved[k] for k in (3, 5, 6, 7, 15)), moved + + +def test_the_supports_are_one_clones_at_three_clones() -> None: + """At three clones, where dividing a weight by K rounds, each support + band and whether the draw is cap-bound are one clone's at every rate: + clone 0's walk compares one clone's prefixes to rounding, the supports + are counted in clone 0's people in true source households, and the cap + bound compares the eligible weight and the target over every clone, + which are one clone's to rounding. Without person_clone_zero the + supports are counted where the clone index is 0; the flagged sources + counted over every clone would read another band at some rates.""" + + base, weights = _clone_base() + sources = base["person_household_id"].to_numpy() + person, clone_weights, index, clone_sources = _as_clones(base, weights, 3) + arguments = {"person_weights": clone_weights, "person_sources": clone_sources} + bands_over_every_clone_differ = 0 + for rate in _CLONE_RATES: + one = derive_is_apprentice( + base, rate=rate, person_weights=weights, person_sources=sources + ) + many = derive_is_apprentice( + person, rate=rate, **arguments, person_clone_index=index + ) + for key in _K_INVARIANT_FIELDS: + assert many.receipt[key] == one.receipt[key], (rate, key) + explicit = derive_is_apprentice( + person, + rate=rate, + **arguments, + person_clone_zero=index == 0, + person_clone_index=index, + ) + assert explicit.receipt == many.receipt + over_every_clone = support_band( + distinct_count(clone_sources, many.flags, clone_weights) + ) + bands_over_every_clone_differ += ( + over_every_clone != one.receipt["flagged_support"] + ) + assert bands_over_every_clone_differ + + +def _walk_before_clones( + person: pd.DataFrame, *, rate: float, weights: np.ndarray +) -> tuple[np.ndarray, dict[str, object]]: + """The flags and the receipt's figures as derive_is_apprentice took them + before the target was met within each clone: one walk over every row.""" + + age = pd.to_numeric(person["age"], errors="coerce").to_numpy(dtype=float) + in_ages = (age >= 21) & (age < 25) + eligible = in_ages & ( + person["employment_status"] + .astype(str) + .isin(("FT_EMPLOYED", "PT_EMPLOYED")) + .to_numpy() + ) + draws = stable_identity_uniforms( + person["person_id"].to_numpy(), seed=0, salt=APPRENTICE_OUTPUT + ) + population_mass = float(weights[in_ages].sum()) + eligible_mass = float(weights[eligible].sum()) + target = float(rate) * population_mass + threshold = frs_person_draws._nearest_target_threshold( + target, draws=draws[eligible], weights=weights[eligible] + ) + flags = eligible & (draws < threshold) + realized = float(weights[flags].sum()) + return flags, { + "population_mass": population_mass, + "eligible_mass": eligible_mass, + "target": target, + "cap_bound": bool(target > 0.0 and eligible_mass <= target), + "realized_mass": realized, + "realization_deviation": realized - target, + } + + +def _bits(value: object) -> object: + return np.float64(value).tobytes() if isinstance(value, float) else value + + +@pytest.mark.parametrize("seed", range(4)) +def test_one_clone_walks_bit_for_bit_as_before(seed: int) -> None: + """Without a clone index, and with one of zeros in int64, int8 or uint16, + the flags and the receipt's masses, target, cap_bound, realized mass and + deviation are, bit for bit and type for type, the walk's over every row + as before the count was taken within each clone, on invented frames of + every size up to eighty people, with zero weights among them, at random + rates.""" + + rng = np.random.default_rng(seed) + compared: dict[str, int] = {} + for _ in range(50): + n = int(rng.integers(0, 81)) + ids = rng.choice(10**6, size=n, replace=False) + 1 + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "age": rng.integers(18, 31, size=n), + "employment_status": rng.choice(_STATUSES, size=n), + } + ) + weights = rng.uniform(0.0, 3000.0, size=n) + weights[rng.random(n) < 0.1] = 0.0 + rate = float(rng.uniform()) + flags, before = _walk_before_clones(person, rate=rate, weights=weights) + for clone_index in ( + None, + np.zeros(n, dtype=np.int64), + np.zeros(n, dtype=np.int8), + np.zeros(n, dtype=np.uint16), + ): + draw = derive_is_apprentice( + person, + rate=rate, + person_weights=weights, + person_sources=ids, + person_clone_index=clone_index, + ) + assert draw.flags.dtype == flags.dtype + assert draw.flags.tobytes() == flags.tobytes() + for key, value in before.items(): + got = draw.receipt[key] + if got == UK_WITHHELD: + continue + compared[key] = compared.get(key, 0) + 1 + assert type(got) is type(value), key + assert _bits(got) == _bits(value), key + assert set(compared) == set(before) and min(compared.values()) > 50, compared + + +def test_the_stage_walks_each_geographic_clone(no_engine) -> None: + """The stage hands the walk each person's household clone: on the + clone-first stand-in at four clones, each clone flags 480 employees, its + own target, clone 0 the one-clone frame's, and the receipt's supports are + one clone's. One walk over every clone would flag others in clone 0.""" + + stage = UKFRSPersonDrawsStageTransform(contract=_Contract(), stage=None) + one_clone = stage(clone_first_frame(_two_household_frame(), 1)).table("person") + one_receipt = stage.checkpoint_metadata()["evidence"]["apprentice_receipt"] + frame = clone_first_frame(_two_household_frame(), 4) + four_clones = stage(frame).table("person") + four_receipt = stage.checkpoint_metadata()["evidence"]["apprentice_receipt"] + clone = (four_clones["person_id"] // GEOGRAPHIC_RADIX).to_numpy() + clone_zero = clone == 0 + flags = four_clones[APPRENTICE_OUTPUT].to_numpy() + assert flags[clone_zero].tolist() == one_clone[APPRENTICE_OUTPUT].tolist() + assert [int(flags[clone == k].sum()) for k in range(4)] == [480] * 4 + for key in _K_INVARIANT_FIELDS: + assert four_receipt[key] == one_receipt[key], key + pooled = derive_is_apprentice( + frame.table("person"), + rate=_Contract.rates[APPRENTICE_RATE_KEY], + person_weights=frs_person_draws.uk_person_household_weights(frame), + person_sources=own_source_households(frame.table("person")), + ) + assert pooled.flags[clone_zero].tolist() != flags[clone_zero].tolist() + + +def test_refuses_a_clone_index_it_cannot_walk_within() -> None: + person = _small([22] * 6, ["FT_EMPLOYED"] * 6) + index = np.array([0, 0, 0, 1, 1, 1], dtype=np.int64) + for options, error, message in ( + ( + {"person_clone_index": index[:-1]}, + ValueError, + "person_clone_index must hold one clone per person", + ), + ( + {"person_clone_index": index.astype(float)}, + TypeError, + "person_clone_index must hold an integer clone per person", + ), + ( + {"person_clone_index": index, "person_clone_zero": index == 1}, + ValueError, + "person_clone_zero must flag exactly the people whose", + ), + ): + with pytest.raises(error, match=message) as refused: + _apprentices(person, rate=0.5, **options) + assert not any(character.isdigit() for character in str(refused.value)) + + def test_refuses_a_frame_without_employment_status() -> None: person = pd.DataFrame({"person_id": [1], "person_household_id": [1], "age": [22]}) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py index a14e867759..63fc60291c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_clone_first_nesting.py @@ -3,9 +3,10 @@ Invented households go through the real geographic expansion (G1, ``geographic_expand``), then the SPI support channel (stage 16), the SPI income band donors (stage 17) and the CGT support split (stage 42), each run by its -stage transform under the packaged stage declaration, G1's included. The -frame runs at K = 1, 2, 3, 7 and 15 geographic clones; it is small, so K = 15 -takes well under a second. Each K is held to K = 1: +stage transform under the packaged stage declaration, G1's included. Other +invented households go through G1, then frs_take_up and frs_person_draws, the +same way. Each frame runs at K = 1, 2, 3, 7 and 15 geographic clones; they are +small, so K = 15 takes well under a second. Each K is held to K = 1: - **Reported first:** stage 17's mass scale at K = 1 and at each K. Clone 0's stage-17 weights nest on K = 1's only while the scale is the one a one-clone @@ -13,6 +14,13 @@ - **Exact:** - clone 0 of G1's output is the one-clone frame, ids and G1's columns included; + - clone 0 of frs_take_up's and of frs_person_draws' output is the one-clone + frame. Their anchored residual and apprentice walk count within each + clone, and every other draw of the two is keyed on the row's own id. The + walk compares weights summed in draw order, which dividing by K scales + exactly at K = 2; on this frame it stops far from any near-tie, so it is + exact at every K. Counted once over every clone instead, each of the + four draws would move clone 0 at some K; - stage 16 copies, and stage 17 seats, K = 1's households in clone 0; - every table of every frame carries its clone index with exactly K values, and every copy carries its parent's. @@ -42,12 +50,17 @@ columns, ids and weights, so the tests written on the stand-in hold for G1. - **The draws behind the clones:** frs_take_up, frs_person_draws and frs_household_draws run on every clone, after G1. On the stand-in at K = 3, - each take-up flag's population support and the apprentice draw's - population and eligible supports are K = 1's, since the stages count - support in true source households. Shares, and counts that follow a draw - (the apprentices flagged, how many values a flag takes), are left out, and - the extended hours are certain, so the under-two usage population follows - no draw. + each take-up flag's population support and distinct values, and the + apprentice draw's population, eligible and flagged supports and whether it + is cap-bound, are K = 1's: the stages count support in true source + households and distinct values in clone 0's rows, clone 0 walks as one + clone, and the cap bound compares sums over every clone that are one + clone's to rounding. The shares are left out, since each clone draws its + own flags on its own ids, and so are the targets: most are contract rates, + but the weighted means (the Scottish Child Payment's and Universal Credit + childcare's) are summed over K times the rows and are K = 1's only to + rounding. The extended hours are certain, so the under-two usage + population follows no draw. Invented values only. Assertion messages give K, stages, counts and booleans, never a weight or an id. @@ -68,9 +81,11 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import assign_binary_with_anchored_residual from microcosm.build.uk_runtime import ( cgt_support, frs_person_draws, + frs_take_up, spi_band_donors, spi_support, ) @@ -90,8 +105,20 @@ from microcosm.build.uk_runtime.frs_household_draws import ( UKFRSHouseholdDrawsStageTransform, ) -from microcosm.build.uk_runtime.frs_person_draws import UKFRSPersonDrawsStageTransform -from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform +from microcosm.build.uk_runtime.frs_person_draws import ( + APPRENTICE_EMPLOYEE_STATUSES, + APPRENTICE_MAX_AGE_EXCLUSIVE, + APPRENTICE_MIN_AGE, + APPRENTICE_OUTPUT, + UKFRSPersonDrawsStageTransform, + derive_is_apprentice, + uk_person_household_weights, +) +from microcosm.build.uk_runtime.frs_take_up import ( + UKFRSTakeUpStageTransform, + aggregate_person_reported_to_benunit, + uc_age_eligible_benunits, +) from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.geographic_expand import ( GEOGRAPHIC_EXPAND_STAGE_NAME, @@ -105,6 +132,7 @@ ) from microcosm.build.uk_runtime.receipt_counts import ( UK_BELOW_TEN_BAND, + own_source_households, support_band, ) from microcosm.build.uk_runtime.spi_band_donors import ( @@ -1080,6 +1108,297 @@ def test_rule_b_puts_the_correction_on_a_copy_at_seven_and_thirteen_clones( assert abs(weights[-1] - values[-1]) < 1e-12 * values[-1] +# --- frs_take_up and frs_person_draws, through G1 ------------------------------------- +# +# Both run after G1, on every clone (microcosm#931). frs_take_up's Child +# Benefit, Pension Credit and Universal Credit flags fill an anchored +# residual, int(rate x units) less the anchors, and frs_person_draws walks +# its apprentices to the rate times the weight aged 21 to 24, each within its +# own clone; every other draw of the two is keyed on the row's own id. +# Invented households: a family unit whose adult is 25 to 64, in work in two +# households of three; a child in every third, some with reported Child +# Benefit; in every other, a unit of one person of an apprentice's age, in +# turn an employee aged 21 and a self-employed person aged 23; a pensioner's +# unit in every fourth, some with reported Pension Credit; and reported +# Universal Credit on every fifth family adult. + +TAKE_UP = "frs_take_up" +PERSON_DRAWS = "frs_person_draws" +#: The stages the invented frame goes through, in spine order. +_DRAW_STAGES = (G1, TAKE_UP, PERSON_DRAWS) +_TAKE_UP_HOUSEHOLDS = 60 +#: Invented design weights' seed. +_TAKE_UP_WEIGHT_SEED = 11 +#: The three anchored draws' rate and the apprentices' rate per head. At +#: these, each of the four draws counted once over every clone moves clone 0 +#: at some K (asserted), so the count within each clone is what nests them. +_ANCHORED_RATE = 0.55 +_APPRENTICE_RATE = 0.4 +#: frs_take_up's anchored draws: each flag, its contract rate and its anchor. +_ANCHORED_DRAWS = { + "would_claim_child_benefit": ("child_benefit", "child_benefit_reported_anchor"), + "would_claim_pc": ("pension_credit", "pension_credit_reported_anchor"), + "would_claim_uc": ("universal_credit", "universal_credit_reported_anchor"), +} +#: How far, relative to its target, the one-clone apprentice walk must stop +#: from a near-tie for its clone-0 flags to be exact at any K: far above the +#: rounding of weights split by K and summed over a frame of this size. +_WALK_MARGIN = 1e-9 + + +class _NestingRatesContract(_Contract): + """test_support's take-up contract at the section's anchored and + apprentice rates.""" + + rates = { + **_Contract.rates, + **{key: _ANCHORED_RATE for key, _ in _ANCHORED_DRAWS.values()}, + "apprentice_stock_19_to_24": _APPRENTICE_RATE, + } + + +def _take_up_frame() -> Frame: + people: list[dict[str, object]] = [] + units: list[dict[str, object]] = [] + households: list[dict[str, object]] = [] + for sernum in range(1, _TAKE_UP_HOUSEHOLDS + 1): + households.append( + {"household_id": sernum, "region": "LONDON" if sernum % 2 else "SCOTLAND"} + ) + # (unit, age, employment status, UC claimant or partner, and the + # Child Benefit, Pension Credit and Universal Credit reported) + members = [ + ( + 1, + 25 + (sernum * 7) % 40, + "FT_EMPLOYED" if sernum % 3 else "UNEMPLOYED", + True, + 0, + 0, + 30 * (sernum % 5 == 0), + ) + ] + if sernum % 3 == 0: + members.append( + (1, sernum % 12, "CHILD", False, 20 * (sernum % 2 == 0), 0, 0) + ) + if sernum % 2 == 0: + employee = sernum % 4 == 0 + members.append( + ( + 2, + 21 if employee else 23, + "FT_EMPLOYED" if employee else "FT_SELF_EMPLOYED", + True, + 0, + 0, + 0, + ) + ) + if sernum % 4 == 1: + members.append( + (3, 68 + sernum % 9, "RETIRED", True, 0, 15 * (sernum % 8 == 1), 0) + ) + units.extend( + { + "benunit_id": sernum * 100 + unit, + "is_married": unit == 1 and sernum % 6 == 0, + } + for unit in sorted({member[0] for member in members}) + ) + for position, (unit, age, status, claimant, cb, pc, uc) in enumerate( + members, start=1 + ): + people.append( + { + "person_id": sernum * 1000 + position, + "person_benunit_id": sernum * 100 + unit, + "person_household_id": sernum, + "age": age, + "employment_status": status, + "months_since_last_birthday": 6.0, + "is_uc_claimant": claimant, + "child_benefit_reported": cb, + "pension_credit_reported": pc, + "universal_credit_reported": uc, + } + ) + return uk_national_frame( + person=pd.DataFrame(people), + benunit=pd.DataFrame(units), + household=pd.DataFrame(households), + household_weights=np.random.default_rng(_TAKE_UP_WEIGHT_SEED).uniform( + 150.0, 1_500.0, _TAKE_UP_HOUSEHOLDS + ), + time_period=_PERIOD, + weight_kind=WeightKind.DESIGN, + ) + + +@pytest.fixture(scope="module") +def draw_runs() -> dict[int, dict[str, Frame]]: + """The invented frame through G1, frs_take_up and frs_person_draws at + K = 1 and at every K of :data:`_CLONE_COUNTS`: each stage's frame.""" + + contract = _NestingRatesContract() + base = _take_up_frame() + runs: dict[int, dict[str, Frame]] = {} + with pytest.MonkeyPatch.context() as patch: + # The apprentices' relief reads an installed engine; their draw does not. + patch.setattr(frs_person_draws, "_installed_uk_system", lambda: None) + for clones in (1, *_CLONE_COUNTS): + transforms = ( + UKGeographicExpandStageTransform(_packaged_stage(G1), clones=clones), + UKFRSTakeUpStageTransform( + contract=contract, + stage=_packaged_stage(TAKE_UP), + population_policy=_POLICY, + ), + UKFRSPersonDrawsStageTransform( + contract=contract, stage=_packaged_stage(PERSON_DRAWS) + ), + ) + frame = base + runs[clones] = {} + for stage, transform in zip(_DRAW_STAGES, transforms, strict=True): + frame = transform(frame) + runs[clones][stage] = frame + return runs + + +def _differing_columns(one: Frame, many: Frame) -> list[str]: + """The columns on which clone 0 of ``many`` is not ``one``: every table's, + by name, after its rows and column order are checked.""" + + differing = [] + for entity in UK_NATIONAL_SCHEMA.entities: + reference = one.table(entity) + table = many.table(entity) + clone_zero = table.loc[table[ladder_clone_index_column(entity)].eq(0)] + assert clone_zero.index.tolist() == list(range(len(reference))), entity + assert list(clone_zero.columns) == list(reference.columns), entity + differing += [ + f"{entity}.{column}" + for column in reference.columns + if not clone_zero[column].equals(reference[column]) + ] + return differing + + +def _apprentice_ages_and_employees( + person: pd.DataFrame, +) -> tuple[np.ndarray, np.ndarray]: + """The people aged 21 to 24, and the employees among them the walk can + flag.""" + + age = person["age"].to_numpy() + in_ages = (age >= APPRENTICE_MIN_AGE) & (age < APPRENTICE_MAX_AGE_EXCLUSIVE) + eligible = ( + in_ages + & person["employment_status"].isin(APPRENTICE_EMPLOYEE_STATUSES).to_numpy() + ) + return in_ages, eligible + + +def _apprentice_walk_margin(frame: Frame) -> float: + """How far, relative to its target, the apprentice walk on ``frame`` (one + clone) stops from a near-tie: from every point where it would stop at + another prefix, the midpoints between consecutive prefixes' weights in + draw order (no one flagged among them), and from the eligible weight, + past which it flags everyone.""" + + person = frame.table("person") + weights = uk_person_household_weights(frame) + in_ages, eligible = _apprentice_ages_and_employees(person) + target = _APPRENTICE_RATE * float(weights[in_ages].sum()) + uniforms = frs_person_draws._draws( + person["person_id"].to_numpy(), APPRENTICE_OUTPUT + ) + order = np.argsort(uniforms[eligible], kind="stable") + prefixes = np.concatenate(([0.0], np.cumsum(weights[eligible][order]))) + stops = np.concatenate(((prefixes[:-1] + prefixes[1:]) / 2, prefixes[-1:])) + return float(np.min(np.abs(stops - target)) / target) + + +@pytest.mark.parametrize("stage", (TAKE_UP, PERSON_DRAWS)) +@pytest.mark.parametrize("clones", _CLONE_COUNTS) +def test_clone_zero_takes_the_one_clone_draws_of_the_take_up_stages( + draw_runs, clones: int, stage: str +) -> None: + """Clone 0 of frs_take_up's and of frs_person_draws' output is the + one-clone frame, every column and dtype, at every K: + + - **The anchored residual.** Each clone counts ``int(rate x units)`` and + its anchors over its own benefit units, which clone 0 shares with one + clone, ids, anchors and population included. + - **The apprentice walk.** Each clone walks its own employees to the rate + times its own weight aged 21 to 24. Clone 0's weights are one clone's + over K, so its walk compares one clone's prefixes over K: exactly at + K = 2, and here at every K, since the one-clone walk stops + :data:`_WALK_MARGIN` or more from a near-tie (asserted). + - **Every other draw** is keyed on the row's own id. + """ + + one, many = draw_runs[1][stage], draw_runs[clones][stage] + if stage == PERSON_DRAWS: + margin = _apprentice_walk_margin(draw_runs[1][TAKE_UP]) + assert margin >= _WALK_MARGIN, "The one-clone apprentice walk is near a tie." + person = one.table("person") + _, employees = _apprentice_ages_and_employees(person) + flagged = int(person[APPRENTICE_OUTPUT].sum()) + assert 0 < flagged < int(employees.sum()), ( + "The walk flags no one or every employee." + ) + differing = _differing_columns(one, many) + assert not differing, f"Clone 0 after {stage} at K={clones} differs on {differing}." + + +def test_the_draws_counted_over_every_clone_would_move_clone_zero(draw_runs) -> None: + """Not vacuous: frs_take_up's three anchored draws counted once over every + clone's units, the way the stage counted before it took the count within + each clone, and the apprentice walk over every clone's employees at once, + each move clone 0 off the one-clone flags at some K. So the count within + each clone is what nests them.""" + + contract = _NestingRatesContract() + one = draw_runs[1][PERSON_DRAWS] + reference = { + "benunit": one.table("benunit").set_index("benunit_id"), + "person": one.table("person").set_index("person_id"), + } + moved: Counter[str] = Counter() + for clones in _CLONE_COUNTS: + expanded = draw_runs[clones][G1] + person, benunit = expanded.table("person"), expanded.table("benunit") + anchors = aggregate_person_reported_to_benunit(person, benunit) + population = uc_age_eligible_benunits(person, benunit, _POLICY) + ids = benunit["benunit_id"].to_numpy() + for flag, (key, anchor) in _ANCHORED_DRAWS.items(): + pooled = assign_binary_with_anchored_residual( + frs_take_up._draws(ids, flag), + contract.rate(key), + anchors[anchor].to_numpy(dtype=bool), + population=population if flag == "would_claim_uc" else None, + ) + expected = reference["benunit"][flag] + pooled = pd.Series(pooled, index=ids).loc[expected.index] + moved[flag] += not pooled.equals(expected.astype(bool)) + drawn = draw_runs[clones][TAKE_UP] + walk = derive_is_apprentice( + drawn.table("person"), + rate=_APPRENTICE_RATE, + person_weights=uk_person_household_weights(drawn), + person_sources=own_source_households(drawn.table("person")), + ) + expected = reference["person"][APPRENTICE_OUTPUT] + pooled = pd.Series(walk.flags, index=drawn.table("person")["person_id"]) + moved[APPRENTICE_OUTPUT] += not pooled.loc[expected.index].equals(expected) + assert set(moved) == {*_ANCHORED_DRAWS, APPRENTICE_OUTPUT} + assert all(moved.values()), ( + f"Draws whose pooled count never moves clone 0: {moved}." + ) + + # --- the draws behind the clones ---------------------------------------------------- # # frs_take_up, frs_person_draws and frs_household_draws run on every clone, @@ -1091,20 +1410,38 @@ def test_rule_b_puts_the_correction_on_a_copy_at_seven_and_thirteen_clones( _DRAW_FAMILIES = 4 _DRAW_PENSIONERS = 4 -#: Receipt fields that are not counts: the flags' shares, and how many values -#: a flag takes, which grows where clones draw apart on their own ids. -_DRAW_SHARE_FIELDS = frozenset({"target", "weighted_share", "distinct_values"}) +#: Receipt fields measured over every clone: the flags' shares, which each +#: clone's own draws move, and their targets, of which the weighted means +#: (the Scottish Child Payment's and Universal Credit childcare's) are K = 1's +#: only to rounding. How many values a flag takes is counted in clone 0's +#: rows, so it is compared. +_DRAW_SHARE_FIELDS = frozenset({"target", "weighted_share"}) +#: The apprentice receipt's fields that do not move with K: its supports, +#: counted in clone 0's people in true source households, and whether the +#: draw was cap-bound, which compares sums over every clone that are one +#: clone's to rounding. Its masses are sums over every clone. +_APPRENTICE_COUNT_FIELDS = ( + "population_support", + "eligible_support", + "flagged_support", + "cap_bound", +) class _CertainHoursContract(_Contract): - """test_support's take-up contract with the extended hours certain. + """test_support's take-up contract with the extended hours certain, and + an apprentice rate at which the walk flags some of the four employees. The under-two usage flag's population is the children under two in units drawn to claim the extended hours. Each clone draws its own claimants, on its own ids, so below a rate of one that population, and its support, grows with K; at one it is every such child on every clone.""" - rates = {**_Contract.rates, "extended_childcare": 1.0} + rates = { + **_Contract.rates, + "extended_childcare": 1.0, + "apprentice_stock_19_to_24": 0.7, + } def _draws_frame() -> Frame: @@ -1158,12 +1495,13 @@ def _draws_frame() -> Frame: def _draw_counts(frame: Frame) -> dict[str, dict[str, object]]: """frs_take_up, frs_person_draws and frs_household_draws on ``frame``, in - spine order, under their packaged declarations, and the count fields of - their receipts: each take-up flag's population support, with its nesting - count or why it is unmeasured, and the apprentice draw's population and - eligible supports. The flags' shares and the apprentices' masses and - flagged support are left out: the anchored residual and the apprentice - walk pool over the clones, whose draws are keyed on their own ids.""" + spine order, under their packaged declarations, and the fields of their + receipts that do not move with K: each take-up flag's population support + and distinct values, with its nesting count or why it is unmeasured, and + the apprentice draw's supports and whether it was cap-bound. The flags' + shares, their targets and the apprentices' masses are left out: each + clone draws its own flags, and a target or mass summed over every clone + is K = 1's only to rounding.""" contract = _CertainHoursContract() transforms = ( @@ -1196,7 +1534,7 @@ def _draw_counts(frame: Frame) -> dict[str, dict[str, object]]: if "apprentice_receipt" in evidence: receipt = evidence["apprentice_receipt"] counts[evidence["stage"]]["apprentices"] = { - key: receipt[key] for key in ("population_support", "eligible_support") + key: receipt[key] for key in _APPRENTICE_COUNT_FIELDS } return counts @@ -1210,8 +1548,8 @@ def test_the_draws_count_in_source_households_on_every_clone(monkeypatch) -> Non assert _draw_counts(three) == one # Not vacuous: every household flag rests on the eight households, which - # reads "3-9", and so do the apprentices, while three clones' rows would - # read another band. + # reads "3-9", and so do the apprentices, three of whose four employees + # the walk flags, while three clones' rows would read another band. households = len(base.table("household")) assert { row["population_support"] for row in one["frs_household_draws"]["flags"] @@ -1219,6 +1557,8 @@ def test_the_draws_count_in_source_households_on_every_clone(monkeypatch) -> Non assert one["frs_person_draws"]["apprentices"] == { "population_support": UK_BELOW_TEN_BAND, "eligible_support": UK_BELOW_TEN_BAND, + "flagged_support": UK_BELOW_TEN_BAND, + "cap_bound": False, } assert support_band(3 * households) != UK_BELOW_TEN_BAND # Without the lineage, as a node not given it would read the frame, the diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_stochastic.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_stochastic.py index ace195a59b..aad9b1dfd0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_stochastic.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_stochastic.py @@ -10,7 +10,16 @@ draw_household_brmas, load_brma_count_resource, ) +from microcosm.build.uk_runtime.frs_take_up import ( + UK_UNDER_TWO_USAGE_OUTPUT, + geographic_clone_indices, + geographic_clone_zero_rows, +) from microcosm.build.uk_runtime.receipt_counts import own_source_households +from test_support.microcosm_build.uk_clone_first_frames import ( + GEOGRAPHIC_RADIX, + clone_first_frame, +) from test_support.microcosm_build.uk_frs_stochastic import * @@ -513,3 +522,227 @@ def test_uc_childcare_take_up_is_drawn_by_family_type() -> None: contract=_Contract(), uc_age_eligible=uc_age_eligible_benunits(person, benunit, _POLICY), ) + + +# --- the anchored draws within each geographic clone (microcosm#931) --------- + +#: frs_take_up's three anchored draws. +_ANCHORED_FLAGS = ("would_claim_child_benefit", "would_claim_pc", "would_claim_uc") +#: Every anchored rate from 0 to 1 in steps of 0.025. +_ANCHORED_RATES = tuple(step / 40 for step in range(41)) +_ANCHORED_HOUSEHOLDS = 36 + + +class _AnchoredRateContract(_Contract): + """test_support's take-up contract with the three anchored rates at one rate.""" + + def __init__(self, rate: float) -> None: + self.rates = { + **_Contract.rates, + "child_benefit": rate, + "pension_credit": rate, + "universal_credit": rate, + } + + +def _anchored_frame() -> object: + """Invented households for the anchored draws, with ids as frs_spine makes + them. + + Every household has a family unit whose adult, a UC claimant, is aged 22 + to 79 in turn, with a child in every other household; every third has a + second unit of one pensioner. Reported Child Benefit (on children), + Pension Credit (on pensioners) and Universal Credit (on the family + adults, some past the draw's working-age reach) anchor some units.""" + + people: list[dict[str, object]] = [] + units: list[dict[str, object]] = [] + households: list[dict[str, object]] = [] + for sernum in range(1, _ANCHORED_HOUSEHOLDS + 1): + households.append( + { + "household_id": sernum, + "region": "LONDON" if sernum % 2 else "SCOTLAND", + "household_weight": 100.0 + 10.0 * sernum, + } + ) + # (unit, age, UC claimant or partner, CB, PC and UC reported) + members = [(1, 22 + (sernum * 7) % 58, True, 0, 0, 40 * (sernum % 5 == 0))] + if sernum % 2 == 0: + members.append((1, sernum % 15, False, 20 * (sernum % 6 == 0), 0, 0)) + if sernum % 3 == 0: + members.append((2, 70 + sernum % 9, True, 0, 15 * (sernum % 9 == 0), 0)) + units.extend( + { + "benunit_id": sernum * 100 + unit, + "is_married": unit == 1 and sernum % 4 == 0, + } + for unit in sorted({member[0] for member in members}) + ) + for position, (unit, age, claimant, cb, pc, uc) in enumerate(members, 1): + people.append( + { + "person_id": sernum * 1000 + position, + "person_benunit_id": sernum * 100 + unit, + "person_household_id": sernum, + "age": age, + "months_since_last_birthday": 6.0, + "is_uc_claimant": claimant, + "child_benefit_reported": cb, + "pension_credit_reported": pc, + "universal_credit_reported": uc, + } + ) + return uk_national_frame( + person=pd.DataFrame(people), + benunit=pd.DataFrame(units), + household=pd.DataFrame(households), + time_period="2024", + ) + + +def _take_up_transform( + rate: float, stage: SourceStageSpec +) -> UKFRSTakeUpStageTransform: + """frs_take_up's transform under its packaged declaration ``stage``.""" + + return UKFRSTakeUpStageTransform( + contract=_AnchoredRateContract(rate), stage=stage, population_policy=_POLICY + ) + + +def _derived_benunits(frame, rate: float, *, clone: int | None = None) -> pd.DataFrame: + """The take-up draws without a clone index, by benefit-unit id: over every + unit of ``frame``, or over one geographic clone's units alone.""" + + person, benunit = frame.table("person"), frame.table("benunit") + if clone is not None: + benunit = benunit.loc[benunit["benunit_id"] // GEOGRAPHIC_RADIX == clone] + person = person.loc[person["person_benunit_id"].isin(benunit["benunit_id"])] + derived = derive_frs_take_up( + benunit, + anchors=aggregate_person_reported_to_benunit(person, benunit), + contract=_AnchoredRateContract(rate), + uc_age_eligible=uc_age_eligible_benunits(person, benunit, _POLICY), + ) + derived.index = benunit["benunit_id"].to_numpy() + return derived + + +def test_anchored_take_up_counts_within_each_geographic_clone() -> None: + """On the clone-first stand-in at three clones, clone 0 takes the + one-clone frame's Child Benefit, Pension Credit and Universal Credit + flags at every rate of the grid, and every clone takes the flags its own + units take alone: frs_take_up passes each unit's household clone, read + through the person memberships, as the anchored draws' strata. One count + over every clone moves clone 0's flags at some rates of each draw, so + the per-clone count is what nests them.""" + + base = _anchored_frame() + one_clone, three_clones = clone_first_frame(base, 1), clone_first_frame(base, 3) + stage = _take_up_stage() + pooled_moves = dict.fromkeys(_ANCHORED_FLAGS, 0) + for rate in _ANCHORED_RATES: + transform = _take_up_transform(rate, stage) + one = transform(one_clone).table("benunit").set_index("benunit_id") + three = transform(three_clones).table("benunit").set_index("benunit_id") + assert set((three.index // GEOGRAPHIC_RADIX).tolist()) == {0, 1, 2} + clone_zero = three.loc[one.index] + for flag in _ANCHORED_FLAGS: + assert clone_zero[flag].tolist() == one[flag].tolist(), (rate, flag) + for clone in range(3): + alone = _derived_benunits(three_clones, rate, clone=clone) + drawn = three.loc[alone.index] + for flag in _ANCHORED_FLAGS: + assert drawn[flag].tolist() == alone[flag].tolist(), (rate, clone, flag) + pooled = _derived_benunits(three_clones, rate).loc[one.index] + for flag in _ANCHORED_FLAGS: + pooled_moves[flag] += pooled[flag].tolist() != one[flag].tolist() + assert all(pooled_moves.values()), pooled_moves + + +def test_anchored_take_up_is_bitwise_at_one_clone() -> None: + """A one-clone frame, its clone index all 0, takes every frs_take_up + output bit for bit as the frame without the index, whose draws count over + every unit as they did before the count was taken within each clone, at + every rate of the grid; so do clone indices of 0 handed to the draws.""" + + base = _anchored_frame() + one_clone = clone_first_frame(base, 1) + assert geographic_clone_indices(base, "benunit") is None + assert not geographic_clone_indices(one_clone, "benunit").any() + person, benunit = base.table("person"), base.table("benunit") + outputs = ( + *(("benunit", column) for column in FRS_TAKE_UP_OUTPUT_COLUMNS), + ("person", UK_UNDER_TWO_USAGE_OUTPUT), + ) + stage = _take_up_stage() + for rate in _ANCHORED_RATES: + transform = _take_up_transform(rate, stage) + without_index, with_zeros = transform(base), transform(one_clone) + for entity, column in outputs: + assert _same_bytes( + with_zeros.table(entity)[column], without_index.table(entity)[column] + ), (rate, column) + arguments = { + "anchors": aggregate_person_reported_to_benunit(person, benunit), + "contract": _AnchoredRateContract(rate), + "uc_age_eligible": uc_age_eligible_benunits(person, benunit, _POLICY), + } + pooled = derive_frs_take_up(benunit, **arguments) + zeros = derive_frs_take_up( + benunit, **arguments, clone_index=np.zeros(len(benunit), dtype=np.int64) + ) + for column in pooled.columns: + assert _same_bytes(zeros[column], pooled[column]), (rate, column) + + +def _same_bytes(got: pd.Series, expected: pd.Series) -> bool: + got, expected = got.to_numpy(), expected.to_numpy() + return got.dtype == expected.dtype and got.tobytes() == expected.tobytes() + + +def test_geographic_clone_indices_read_each_rows_household_clone() -> None: + """Every row reads its household's clone: a person's household's, a + benefit unit's through its members; a frame without the index has none.""" + + base = _anchored_frame() + three_clones = clone_first_frame(base, 3) + for entity, id_column in ( + ("household", "household_id"), + ("benunit", "benunit_id"), + ("person", "person_id"), + ): + indices = geographic_clone_indices(three_clones, entity) + assert indices.dtype.kind == "i" + np.testing.assert_array_equal( + indices, + three_clones.table(entity)[id_column].to_numpy() // GEOGRAPHIC_RADIX, + ) + np.testing.assert_array_equal( + geographic_clone_zero_rows(three_clones, entity), indices == 0 + ) + assert geographic_clone_indices(base, entity) is None + assert geographic_clone_zero_rows(base, entity).all() + + +def test_take_up_draws_refuse_a_clone_index_they_cannot_count_within() -> None: + frame = clone_first_frame(_anchored_frame(), 3) + person, benunit = frame.table("person"), frame.table("benunit") + arguments = { + "anchors": aggregate_person_reported_to_benunit(person, benunit), + "contract": _AnchoredRateContract(0.5), + "uc_age_eligible": uc_age_eligible_benunits(person, benunit, _POLICY), + } + for clone_index, error, message in ( + ( + np.zeros(len(benunit) - 1, dtype=np.int64), + ValueError, + "clone_index must align", + ), + (np.zeros(len(benunit)), TypeError, "strata must be integers"), + (np.full(len(benunit), np.nan), TypeError, "none missing"), + ): + with pytest.raises(error, match=message) as refused: + derive_frs_take_up(benunit, **arguments, clone_index=clone_index) + assert not any(character.isdigit() for character in str(refused.value)) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py index 3c2d091cd5..0564047338 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_identity_stability_receipts.py @@ -3630,6 +3630,13 @@ def entry(self, key: str): return SimpleNamespace(raw={"entity": "person"}) +class _E4UniversalCreditContract(_E4Contract): + """The scenario's rates, with Universal Credit's at ``universal_credit``.""" + + def __init__(self, universal_credit: float) -> None: + self.rates = {**_E4Contract.rates, "universal_credit": universal_credit} + + class _LHAEngine: """LHA_category by whether the benefit unit claims Universal Credit.""" @@ -3770,8 +3777,9 @@ def _e4_propensity() -> pd.DataFrame: ) -def _e4_chain(clones: int) -> SimpleNamespace: - """The scenario through E4's stages and the later layers; each stage's frame.""" +def _e4_chain(clones: int, contract: _E4Contract | None = None) -> SimpleNamespace: + """The scenario through E4's stages and the later layers; each stage's + frame. ``contract`` defaults to the scenario's rates.""" from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform @@ -3783,7 +3791,11 @@ def _e4_chain(clones: int) -> SimpleNamespace: derive_frs_person_draws, uk_person_household_weights, ) - from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform + from microcosm.build.uk_runtime.frs_take_up import ( + UKFRSTakeUpStageTransform, + geographic_clone_indices, + geographic_clone_zero_rows, + ) from microcosm.build.uk_runtime.geographic_expand import ( expand_uk_geographic_clones, ) @@ -3793,15 +3805,16 @@ def _e4_chain(clones: int) -> SimpleNamespace: ) from microcosm.build.uk_runtime.spi_spine import UKSPISupportChannelStageTransform - contract = _E4Contract() + contract = _E4Contract() if contract is None else contract expanded = expand_uk_geographic_clones(_e4_raw_frame(), clones=clones).frame taken = UKFRSTakeUpStageTransform( contract=contract, stage=_manifest_stage("frs_take_up"), population_policy=_E4_POLICY, )(expanded) - # The person draws through their derivation: the stage transform reads - # the installed engine's apprentice schedule, which only its receipt uses. + # The person draws through their derivation, with what the stage hands + # it: the stage transform reads the installed engine's apprentice + # schedule, which only its receipt uses. person = taken.table("person").copy() drawn = derive_frs_person_draws( person, @@ -3810,6 +3823,8 @@ def _e4_chain(clones: int) -> SimpleNamespace: person_sources=record_source_households( taken, "person", stage="frs_person_draws", before_copies=True ), + person_clone_zero=geographic_clone_zero_rows(taken, "person"), + person_clone_index=geographic_clone_indices(taken, "person"), ) for column in FRS_PERSON_DRAW_OUTPUT_COLUMNS: person[column] = drawn[column].to_numpy() @@ -3952,6 +3967,58 @@ def engine(*, clones=None): assert childcare and childcare <= still_equal assert "would_claim_uc" not in still_equal + @pytest.mark.parametrize("universal_credit", [0.5, 0.4]) + def test_the_recompute_counts_within_each_clone_at_three_clones( + self, universal_credit: float, monkeypatch + ) -> None: + """At three clones frs_take_up's anchored draws and frs_person_draws' + apprentice walk count within each clone (microcosm#931), and E4 hands + both recomputes each row's clone, by id: no stored column differs, the + store still holds the Child Benefit and Pension Credit flags, which no + later layer of the scenario rewrites, and the BRMA stage's + would_claim_uc is restored as the stage drew it. Without the clone + index, as E4 recomputed before, the walk stops at another prefix and + the Child Benefit residual is not the stored one; at a Universal + Credit rate of 0.4, whose int(rate x units) rounds otherwise over + every clone, the BRMA stage's would_claim_uc, and so its BRMA, differ + too.""" + + tool = _load_tool() + built = _e4_chain(3, _E4UniversalCreditContract(universal_credit)) + scoped = tool._frame_as_draw_stages_saw( + built.final, state_pension_age=_E4_POLICY.state_pension_age + ) + stage_time = built.brma.table("benunit").set_index("benunit_id") + + def recompute() -> tuple[dict[str, object], int]: + """The receipt, and how many benefit units the BRMA stage's + restored would_claim_uc puts off the stage's.""" + + engine = _LHAEngine() + receipt = _e4_receipt(tool, scoped, built.contract, engine=engine) + (read,) = engine.frames + benunit = read.table("benunit").set_index("benunit_id") + restored = benunit["would_claim_uc"].to_numpy(dtype=bool) + drawn = stage_time["would_claim_uc"].reindex(benunit.index).to_numpy() + return receipt, int((restored != drawn.astype(bool)).sum()) + + receipt, off = recompute() + assert receipt["matches_stored_columns"] is True + assert receipt["stored_mismatches"] == {} + assert receipt["held_to_tolerance"] == {} + still_equal = set(receipt["rewritten_later_still_equal"]["benunit"]) + assert {"would_claim_child_benefit", "would_claim_pc"} <= still_equal + assert off == 0 + + monkeypatch.setattr(tool, "geographic_clone_indices", lambda *_: None) + pooled, off = recompute() + assert "is_apprentice" in pooled["stored_mismatches"]["person"] + still_equal = set(pooled["rewritten_later_still_equal"]["benunit"]) + assert "would_claim_child_benefit" not in still_equal + if universal_credit == 0.4: + assert off > 0 + assert "brma" in pooled["stored_mismatches"]["household"] + def test_brma_on_the_final_frames_flags_draws_from_other_cells(self) -> None: # The BRMA stage read LHA_category on its own input, carrying # frs_take_up's would_claim_uc and no legacy closure claim. The final @@ -4184,3 +4251,37 @@ def test_the_tolerance_is_relative_to_the_people_aged_21_to_24(self) -> None: assert not tool._apprentice_walk_holds( person, three, person_weights=weights, rate=rate ) + + def test_the_walk_holds_within_each_clone(self) -> None: + # Two clones of the same four employees of weight 1, at a rate of a + # half: frs_person_draws flags each clone's first two in draw order, + # its own target. Given the clone index, those flags hold, and the + # first four in draw order over both clones, one walk over every + # clone, do not; without it, the other way round (microcosm#931). + tool = _load_tool() + one = self._people(4) + person = pd.concat( + [one, one.assign(person_id=one["person_id"] + 10**9)], ignore_index=True + ) + clones = np.repeat(np.array([0, 1], dtype=np.int64), len(one)) + order = self._in_draw_order(person) + each_clone = np.zeros(len(person), dtype=bool) + for clone in (0, 1): + each_clone[[i for i in order if clones[i] == clone][:2]] = True + over_both = np.zeros(len(person), dtype=bool) + over_both[order[:4]] = True + # The premise: the two walks flag different people here. + assert not np.array_equal(each_clone, over_both) + weights = np.ones(len(person)) + assert tool._apprentice_walk_holds( + person, each_clone, person_weights=weights, rate=0.5, clone_index=clones + ) + assert not tool._apprentice_walk_holds( + person, over_both, person_weights=weights, rate=0.5, clone_index=clones + ) + assert tool._apprentice_walk_holds( + person, over_both, person_weights=weights, rate=0.5 + ) + assert not tool._apprentice_walk_holds( + person, each_clone, person_weights=weights, rate=0.5 + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_k_sensitive_constants.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_k_sensitive_constants.py index 2968371341..1f0d485d20 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_k_sensitive_constants.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_k_sensitive_constants.py @@ -32,6 +32,7 @@ regional_home_values, salary_sacrifice, spi_band_donors, + spi_benefit_coherence, spi_employment_shell, spi_rank_transport, stage_health, @@ -46,8 +47,9 @@ #: How the code meets the clone count, each with what it means. TREATMENTS = { "per clone": ( - "declared for one clone: the stage applies it within each clone, and " - "its check holds K times the stage's figure to it" + "declared for one clone: the stage applies it within each clone, and a " + "check that reads it over every clone holds K times the stage's figure " + "to it" ), "invariant": "what it counts does not move with K", "kept": "it moves with K, and is kept as it is for the reason given", @@ -401,17 +403,80 @@ def _source_names(function: Callable[..., object], *names: str) -> bool: ), KSensitiveConstant( name="take-up target count", - where="stochastic_assignment.assign_binary_with_anchored_residual", - read=lambda: _source_names( - stochastic_assignment.assign_binary_with_anchored_residual, - "int(rate * int(population.sum()))", + where=( + "frs_take_up.derive_frs_take_up's Child Benefit, Pension Credit and " + "Universal Credit draws: the int(rate x n) of " + "stochastic_assignment.assign_binary_with_anchored_residual, taken " + "within its strata" ), - value=True, + read=lambda: ( + _source_names( + stochastic_assignment.assign_binary_with_anchored_residual, + "int(rate * int(population.sum()))", + "np.unique(strata)", + ), + inspect.getsource(frs_take_up.derive_frs_take_up).count("strata=strata"), + _source_names( + frs_take_up.add_frs_take_up, "clone_index=geographic_clone_indices" + ), + ), + value=(True, 3, True), + counts="source count", + treatment="per clone", + reason=( + "frs_take_up hands the draws each benefit unit's geographic clone " + "as their strata, so each clone counts its own units, each FRS " + "benefit unit once, and clone 0's count is one clone's" + ), + ), + KSensitiveConstant( + name="SPI take-up target count", + where=( + "spi_benefit_coherence._redraw_spi_uc_take_up's Universal Credit " + "draw: the int(rate x n) of " + "stochastic_assignment.assign_binary_with_anchored_residual" + ), + read=lambda: ( + _source_names( + spi_benefit_coherence._redraw_spi_uc_take_up, + "assign_binary_with_anchored_residual", + ), + _source_names(spi_benefit_coherence._redraw_spi_uc_take_up, "strata"), + ), + value=(True, False), counts="rows", - treatment="later", + treatment="kept", + reason=( + "int(rate x n) rounds once over the SPI units of every clone: the " + "redraw lies outside the stages #931 nests clone 0 on, so it keeps " + "one count over every clone, which its clone index as strata would " + "take per clone" + ), + ), + KSensitiveConstant( + name="apprentice target", + where=( + "frs_person_draws.derive_is_apprentice's walk target, the rate times " + "the weight aged 21 to 24" + ), + read=lambda: ( + _source_names( + frs_person_draws.derive_is_apprentice, + "_clone_rows(person_clone_index", + "weights[in_ages & rows]", + ), + _source_names( + frs_person_draws.add_frs_person_draws_with_receipt, + "person_clone_index=geographic_clone_indices", + ), + ), + value=(True, True), + counts="weight", + treatment="per clone", reason=( - "int(rate x n) rounds once over the rows of every clone; #931 " - "group (c) takes it up with the per-clone take-up counts" + "each clone walks its own employees to the rate times its own " + "weight aged 21 to 24, a 1 / K share, so clone 0's walk is one " + "clone's at its weights over K" ), ), KSensitiveConstant( diff --git a/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py b/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py index 519d986ddc..bf7a24e9bf 100644 --- a/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py +++ b/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py @@ -1,7 +1,7 @@ """The clone-first UK spine at four geographic clones on the H2 fixture. -microcosm#931 group (b). The H2 fixture's invented sources run the whole -spine in this process twice, through the real engine: at one geographic +microcosm#931 groups (b) and (c). The H2 fixture's invented sources run the +whole spine in this process twice, through the real engine: at one geographic clone, the fixture's own config, and at four, the same config with ``geographic_clones`` 4, so the graph's geographic_expand node, the transform that makes the clones and the engine all take K = 4. Each run records, per @@ -17,10 +17,11 @@ geographic_expand on. - **Exact:** clone 0 after geographic_expand and geographic_identity is the one-clone frame on every column, at K x its weight bitwise. Through - frs_brma it differs from K = 1 only where a draw pools over the clones - (frs_take_up's anchored residual, frs_person_draws' apprentice walk) and in - the BRMA of the households those draws move, whose LHA_category the engine - computes from them; so the keyed per-row draws nest. spi_support_channel + frs_brma it is still the one-clone frame on every column: frs_take_up's + anchored residual and frs_person_draws' apprentice walk count within each + clone, the walk on prefixes that 1/4 scales exactly, every other draw is + keyed on the row's own id, and frs_brma's cell reads the LHA_category the + engine computes for clone 0 from that clone's rows. spi_support_channel copies K = 1's households in clone 0, at nesting weights, and spi_income_band_donors seats K = 1's households in K = 1's bands with K = 1's carriers. @@ -32,10 +33,9 @@ the stratum's own: K x clone-0 weights nest on one clone's only where the two scales are equal. Whether the stage takes a scale per clone is open (microcosm#931). -- **Tolerance classes:** the pooled draws keep their K-invariant receipt - fields; from frs_take_up on, every output every clone carries, at a - weighted share near one clone's, and every node keeps K = 1's household - mass. +- **Tolerance classes:** the draws keep their K-invariant receipt fields; + from frs_take_up on, every output every clone carries, at a weighted share + near one clone's, and every node keeps K = 1's household mass. - **Receipts:** the clone count in geographic_expand's, spi_support_channel's and spi_income_band_donors' receipts and, at K = 4 only, the support clips' and property components'; the geographic fold receipts at the six folding @@ -113,13 +113,14 @@ #: which is the first engine call after G1. _FIRST_DRAW = "frs_take_up" _LAST_DRAW = "frs_brma" -#: The draws that pool over the clones before spi_support_channel: -#: frs_take_up's anchored residual takes ``int(rate x units)`` of the pool's -#: units, and frs_person_draws' apprentice walk meets a weighted target over -#: the pool, so clone 0's draws need not be one clone's. Every other draw from +#: The draws that count within each geographic clone before +#: spi_support_channel: frs_take_up's anchored residual takes +#: ``int(rate x units)`` of each clone's units, and frs_person_draws' +#: apprentice walk meets the rate times each clone's weight aged 21 to 24, so +#: clone 0's draws are one clone's (microcosm#931). Every other draw from #: frs_take_up to frs_brma is keyed on the row's own id, which clone 0 shares #: with one clone. -_POOLED_DRAWS = { +_COUNTED_WITHIN_EACH_CLONE = { ("benunit", "would_claim_child_benefit"), ("benunit", "would_claim_pc"), ("benunit", "would_claim_uc"), @@ -620,39 +621,22 @@ def test_clone_zero_after_the_clones_is_the_one_clone_frame(runs, stage: str) -> assert (weights * _CLONES).tobytes() == _weights(one).to_numpy().tobytes() -def _households_off_one_clone_on_the_pooled_draws(one: Frame, many: Frame) -> set[int]: - """The households where a clone-0 pooled draw (``_POOLED_DRAWS``) is not - one clone's: a benefit unit's take-up flag or a member's apprentice flag. - frs_brma takes each unit's count-table cell from the engine's - LHA_category, which reads the household's members and units (a claim to - Universal Credit decides whose non-dependants count), so only in these - households may clone 0's BRMA lie in another cell.""" +def test_clone_zero_through_frs_brma_is_the_one_clone_frame(runs) -> None: + """Through frs_brma clone 0 is the one-clone frame on every column, every + draw from frs_take_up to frs_brma among them: - person = one.table("person") - homes = { - "person": person.set_index("person_id")["person_household_id"], - "benunit": person.groupby("person_benunit_id")["person_household_id"].first(), - } - ids = {"person": "person_id", "benunit": "benunit_id"} - off: set[int] = set() - for entity, column in sorted(_POOLED_DRAWS): - left = one.table(entity).set_index(ids[entity])[column] - right = ( - _clone_zero(many, entity).set_index(ids[entity])[column].reindex(left.index) - ) - moved = left.index[left.to_numpy() != right.to_numpy()] - off |= set(homes[entity].loc[moved].tolist()) - return off - - -def test_clone_zero_keeps_the_keyed_per_row_draws_through_frs_brma(runs) -> None: - """Through frs_brma clone 0 is the one-clone frame on every column but - the draws that pool over the clones (``_POOLED_DRAWS``) and the BRMA of - the households where those differ: every other draw from frs_take_up to - frs_brma is keyed on the row's own id, and the BRMA draw's cell reads the - LHA_category the batched engine computes for clone 0 from that clone's - rows alone, so it is one clone's wherever the household's inputs are. - The other households' BRMA is left to the share class.""" + - frs_take_up's anchored residual and frs_person_draws' apprentice walk + count within each clone (``_COUNTED_WITHIN_EACH_CLONE``); the walk + compares prefixes of clone 0's weights, one clone's over 4, which the + split scales exactly. + - Every other draw is keyed on the row's own id, which clone 0 shares + with one clone. + - frs_brma takes each unit's count-table cell from the engine's + LHA_category, which reads the household's members and units (a claim + to Universal Credit decides whose non-dependants count). The batched + engine computes it for clone 0 from that clone's rows alone, which are + one clone's, so the BRMA is one clone's too. + """ one, many = runs[1].frames[_LAST_DRAW], runs[_CLONES].frames[_LAST_DRAW] compared: set[tuple[str, str]] = set() @@ -662,37 +646,24 @@ def test_clone_zero_keeps_the_keyed_per_row_draws_through_frs_brma(runs) -> None clone_zero = _clone_zero(many, entity).reset_index(drop=True) assert list(clone_zero.columns) == list(reference.columns), entity for column in reference.columns: - if (entity, column) in _POOLED_DRAWS or (entity, column) == _BRMA: - continue compared.add((entity, column)) if not clone_zero[column].equals(reference[column]): differing.append(f"{entity}.{column}") assert not differing, f"Clone 0 after {_LAST_DRAW} differs on {differing}." - moved = _households_off_one_clone_on_the_pooled_draws(one, many) - entity, column = _BRMA - reference = one.table(entity).set_index("household_id")[column] - clone_zero = ( - _clone_zero(many, entity) - .set_index("household_id")[column] - .reindex(reference.index) - ) - kept = ~reference.index.isin(list(moved)) - assert kept.any(), "The pooled draws moved every household; no BRMA compared." - off = int((clone_zero.to_numpy()[kept] != reference.to_numpy()[kept]).sum()) - assert off == 0, ( - f"{off} household(s) whose pooled draws are one clone's draw another " - f"BRMA in clone 0 ({len(moved)} household(s) left to the share class)." - ) - compared.add(_BRMA) stages = runs[1].stage_names - keyed = { + drawn = { (entity, column) - for stage in ("frs_person_draws", "frs_household_draws", _LAST_DRAW) + for stage in ( + _FIRST_DRAW, + "frs_person_draws", + "frs_household_draws", + _LAST_DRAW, + ) for column in _stage_outputs(stages, stage) for entity in UK_NATIONAL_SCHEMA.entities if column in one.table(entity).columns - } - _POOLED_DRAWS - assert keyed and keyed <= compared + } + assert _COUNTED_WITHIN_EACH_CLONE | {_BRMA} <= drawn <= compared def _stage_outputs(stage_names: tuple[str, ...], stage: str) -> tuple[str, ...]: @@ -822,12 +793,14 @@ def test_the_band_donors_fund_each_clone_zero_stratum_at_one_ratio(runs) -> None # --- tolerance classes --------------------------------------------------------------- -def test_the_pooled_draws_keep_their_k_invariant_receipt_fields(runs) -> None: +def test_the_draws_keep_their_k_invariant_receipt_fields(runs) -> None: """frs_take_up, frs_person_draws and frs_household_draws count each flag's population in true sources, and frs_person_draws' apprentice walk - its target, masses and supports, so these are one clone's at K = 4; each + its target, masses and supports, so these are one clone's at K = 4; the + flagged support too, counted in clone 0, whose walk is one clone's. Each target and mass, summed over the clones' rows, is one clone's to its - summation order. The realized shares are left to the share class.""" + summation order. The realized shares and mass are left to the share + class: the other clones flag their own employees.""" one, many = runs[1].evidence, runs[_CLONES].evidence for stage in (_FIRST_DRAW, "frs_person_draws", "frs_household_draws"): @@ -855,6 +828,7 @@ def test_the_pooled_draws_keep_their_k_invariant_receipt_fields(runs) -> None: "eligible_mass", "population_support", "eligible_support", + "flagged_support", "cap_bound", ): expected, found = apprentices["one"][field], apprentices["many"][field] diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index d08e4b4da9..5c7bc399c1 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -38,6 +38,7 @@ aggregate_person_reported_to_benunit, derive_frs_take_up, derive_under_two_childcare_usage, + geographic_clone_indices, person_unit_flag, uc_age_eligible_benunits, uk_take_up_population_policy, @@ -120,6 +121,12 @@ def e4_identity_receipt( take-up draw used (``uk_take_up_population_policy``); it is read from the engine when not supplied, so a hermetic caller passes one. + frs_take_up's anchored draws and frs_person_draws' apprentice walk count + within each geographic clone (microcosm#931), so both recomputes take each + row's clone as the stages read it (``geographic_clone_indices``), carried + by entity id onto the permuted tables; a frame without the clone index is + one clone. + ``lha_category`` is the BRMA draw's predictor, aligned to the benefit-unit rows. Without it, ``engine`` materializes it on the frame the BRMA stage read, as far as the artifact restores it (``_brma_stage_lha_category``): @@ -136,8 +143,9 @@ def e4_identity_receipt( it and, under ``rewritten_later_still_equal``, the columns the store still holds bitwise (a later stage may write only rows the scope drops, such as copies). ``is_apprentice`` is held exactly, or where it differs, - to its walk on the restored weights (``_apprentice_walk_holds``), since - those are the stage's to rounding only; the receipt names it then. + to its walk within each clone on the restored weights + (``_apprentice_walk_holds``), since those are the stage's to rounding + only; the receipt names it then. """ from microcosm.build.uk_runtime.frs_person_draws import ( @@ -161,6 +169,11 @@ def e4_identity_receipt( strict=True, ) ) + # Each benefit unit's and person's geographic clone, by id, which a row + # permutation carries with each id. + clone_by_id = { + entity: _clone_index_by_id(frame, entity) for entity in ("benunit", "person") + } def draws(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: """The take-up, person and household draws, keyed by entity id.""" @@ -173,6 +186,7 @@ def draws(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: uc_age_eligible=uc_age_eligible_benunits( person_t, benunit_t, population_policy ), + clone_index=_clones_on(clone_by_id["benunit"], benunit_t["benunit_id"]), ) claiming = person_unit_flag( person_t, @@ -189,6 +203,7 @@ def draws(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: # Only the draws are compared, never the receipt's support # counts, so each row's household stands as its source. person_sources=own_source_households(person_t), + person_clone_index=_clones_on(clone_by_id["person"], person_t["person_id"]), ) person_draws[UK_UNDER_TWO_USAGE_OUTPUT] = derive_under_two_childcare_usage( person_t, claiming=claiming, contract=contract @@ -293,6 +308,7 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: .map(weight_by_household) .to_numpy(dtype=float), rate=contract.rate(APPRENTICE_RATE_KEY, contract.build_year), + clone_index=_clones_on(clone_by_id["person"], person["person_id"]), ) ): held_to_tolerance.setdefault(entity, []).append(column) @@ -315,13 +331,15 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: "not compared (rewritten_later names the stages, and " "rewritten_later_still_equal the columns the store still holds " "bitwise), and is_apprentice, which where it differs is held to " - "its walk on the " + "its walk within each geographic clone on the " "restored stage-time weights (they are the stage's to rounding): " - "the stored flags must be exactly the eligible employees whose " - "keyed uniform lies at or below the largest flagged one's, and " - "their weight must lie as near the target as the recompute's, to " + "in each clone the stored flags must be exactly the clone's " + "eligible employees whose keyed uniform lies at or below the " + "largest flagged one's, and their weight must lie as near the " + "clone's target as the recompute's, to " f"a relative {_APPRENTICE_WALK_RELATIVE_TOLERANCE:g} of the weight " - "of the people aged 21 to 24 (held_to_tolerance names it then); " + "of the clone's people aged 21 to 24 (held_to_tolerance names it " + "then); " "brma: bitwise, on LHA_category materialized on the frame frs_brma " "read as far as the artifact restores it (lha_category_scope)" ), @@ -444,21 +462,24 @@ def _apprentice_walk_holds( *, person_weights: np.ndarray, rate: float, + clone_index: np.ndarray | None = None, ) -> bool: """Whether stored ``is_apprentice`` flags are the stage's walk, to rounding. - ``frs_person_draws`` flags the eligible employees aged 21 to 24 in the - order of their identity-keyed uniform until their weight lies nearest - ``rate`` times the weight of everyone aged 21 to 24 - (``derive_is_apprentice``). ``person_weights`` are each person's household - weight restored to the stage's to rounding (``_frame_as_draw_stages_saw``), - so where two prefixes of the walk lie within rounding of the target a + ``frs_person_draws`` flags the eligible employees aged 21 to 24 of each + geographic clone (``clone_index``, aligned with ``person``; everyone in + one clone when not given) in the order of their identity-keyed uniform + until their weight lies nearest ``rate`` times the weight of the clone's + people aged 21 to 24 (``derive_is_apprentice``). ``person_weights`` are + each person's household weight restored to the stage's to rounding + (``_frame_as_draw_stages_saw``), so where a clone's target lies within + rounding of the midpoint between two consecutive prefixes of its walk a recompute can flag one person more or fewer than the stage did. The - stored flags hold where they are exactly the eligible employees whose - uniform lies at or below the largest flagged one's, and their weight lies - as near the target as the recompute's, to - :data:`_APPRENTICE_WALK_RELATIVE_TOLERANCE` of the weight of everyone - aged 21 to 24. + stored flags hold where, in every clone, they are exactly the clone's + eligible employees whose uniform lies at or below the largest flagged + one's, and their weight lies as near the clone's target as the + recompute's, to :data:`_APPRENTICE_WALK_RELATIVE_TOLERANCE` of the weight + of the clone's people aged 21 to 24. """ from microcosm.build.uk_runtime.frs_person_draws import ( @@ -466,6 +487,7 @@ def _apprentice_walk_holds( APPRENTICE_MAX_AGE_EXCLUSIVE, APPRENTICE_MIN_AGE, APPRENTICE_OUTPUT, + _clone_rows, _draws, _nearest_target_threshold, ) @@ -483,20 +505,44 @@ def _apprentice_walk_holds( uniforms = _draws(person["person_id"].to_numpy(), APPRENTICE_OUTPUT) if (flags & ~eligible).any(): return False - if flags.any() and not np.array_equal( - flags, eligible & (uniforms <= uniforms[flags].max()) - ): - return False - population = float(weights[in_ages].sum()) - target = float(rate) * population - threshold = _nearest_target_threshold( - target, draws=uniforms[eligible], weights=weights[eligible] - ) - walked = float(weights[eligible & (uniforms < threshold)].sum()) - stored = float(weights[flags].sum()) - return abs(stored - target) <= ( - abs(walked - target) + _APPRENTICE_WALK_RELATIVE_TOLERANCE * population - ) + for rows in _clone_rows(clone_index, len(person)): + clone_flags = flags & rows + clone_eligible = eligible & rows + if clone_flags.any() and not np.array_equal( + clone_flags, clone_eligible & (uniforms <= uniforms[clone_flags].max()) + ): + return False + population = float(weights[in_ages & rows].sum()) + target = float(rate) * population + threshold = _nearest_target_threshold( + target, draws=uniforms[clone_eligible], weights=weights[clone_eligible] + ) + walked = float(weights[clone_eligible & (uniforms < threshold)].sum()) + stored = float(weights[clone_flags].sum()) + if not abs(stored - target) <= ( + abs(walked - target) + _APPRENTICE_WALK_RELATIVE_TOLERANCE * population + ): + return False + return True + + +def _clone_index_by_id(frame, entity: str) -> pd.Series | None: + """Each ``entity`` row's geographic clone, by its id, as frs_take_up and + frs_person_draws read it (``geographic_clone_indices``), or None on a + frame without the clone index (one clone).""" + + indices = geographic_clone_indices(frame, entity) + if indices is None: + return None + return pd.Series(indices, index=frame.table(entity)[f"{entity}_id"].to_numpy()) + + +def _clones_on(clone_by_id: pd.Series | None, ids: pd.Series) -> np.ndarray | None: + """``clone_by_id`` on the rows ``ids`` name, in their order, or None.""" + + if clone_by_id is None: + return None + return clone_by_id.loc[ids.to_numpy()].to_numpy() def e5_identity_receipt( @@ -2867,11 +2913,12 @@ def main() -> int: # E4's stages drew on every geographic clone of the raw FRS rows, # before the SPI rows were stacked, and its take-up targets are - # unweighted int(rate * n_units): recompute on the survey channel - # only, or every target moves with the synthetic rows (the post-#717 - # scoping rule the E5 receipt already applies), at the weights the - # stages read, as far as the artifact restores them. The engine runs - # clone by clone at the spine's clone count, as the stages' did. + # unweighted int(rate * n_units) within each clone: recompute on the + # survey channel only, or every target moves with the synthetic rows + # (the post-#717 scoping rule the E5 receipt already applies), at the + # weights the stages read, as far as the artifact restores them. The + # engine runs clone by clone at the spine's clone count, as the + # stages' did. population_policy = uk_take_up_population_policy(uk_time_period(frame)) receipt = e4_identity_receipt( _frame_as_draw_stages_saw( @@ -3001,9 +3048,9 @@ def main() -> int: #: (``_frame_as_draw_stages_saw``). _SPI_SUPPORT_STAGE = "spi_support_channel" _SPI_PRIOR_MASS_OPERATION = "allocate_zero_weight_prior_mass" -#: How far, relative to the weight of everyone aged 21 to 24, stored -#: is_apprentice flags may lie further from the walk's target than the -#: recompute's on the restored weights (``_apprentice_walk_holds``): the +#: How far, relative to the weight of a clone's people aged 21 to 24, a +#: clone's stored is_apprentice flags may lie further from its target than +#: the recompute's on the restored weights (``_apprentice_walk_holds``): the #: restored weights are the stage's to a few floats per row, and to the #: exact-total corrections of two stages, far inside it. _APPRENTICE_WALK_RELATIVE_TOLERANCE = 1e-9 From 725b9a53c161e2f489412d1615169c632e89257a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:09:28 +0000 Subject: [PATCH 067/108] Run the clone-first H2 test after merge only (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit María ruled on 11 Nov that the clone-first H2 engine test, which (b) commit 5 put in the engine-uk job at about 27 more minutes per leg, should not hold up every UK pull request. It moves to a new test category, engine_after_merge/uk/, registered in tools/ci_test_plan.py as the engine-after-merge-uk group of a new job, engine-uk-after-merge. That job runs on pushes to main only (if: github.event_name == 'push'), on both Python legs, with a 50-minute cap, about 1.6x its expected leg. The engine-uk job's cap and comment go back to main's: 60 minutes. The module is renamed test_h2_clone_first.py. As test_acceptance_h_clone_first.py it fell under the graph acceptance burndown's suite (packages/microcosm-graph/tests/**/test_acceptance_*.py), whose honesty checks it fails: a helper class, pytest referenced outside a mark decorator, and getattr on a module. So the lint job's burndown verification would have failed once this PR gets CI. It is not a charter test, and the burndown now passes. tools/run_engine_uk_tests.sh takes the job whose files it runs, engine-uk by default. docs/agent-guide.md lists the new category, and a changelog fragment describes the job. Tests: the registry case for engine_after_merge/uk/, and that only the after-merge job waits for a push to main. Checks: ci_test_plan.py verify (the new group holds the one module), the burndown verification, test_ci_test_plan.py and test_ci_integration_job.py (46 passed), and the moved module collects its 13 tests. Its content is unchanged apart from one docstring paragraph, so it was not run again. No node key moves. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 32 +++++++++++++++---- .../931-engine-uk-after-merge-ci.added.md | 1 + docs/agent-guide.md | 3 ++ .../shared/test_ci_integration_job.py | 16 ++++++++++ .../engine_free/shared/test_ci_test_plan.py | 8 +++++ .../uk/test_h2_clone_first.py} | 3 ++ tools/ci_test_plan.py | 8 +++++ tools/run_engine_uk_tests.sh | 4 ++- 8 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 changelog.d/931-engine-uk-after-merge-ci.added.md rename packages/microcosm-graph/tests/{engine/uk/test_acceptance_h_clone_first.py => engine_after_merge/uk/test_h2_clone_first.py} (99%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 460ce9385d..abf79aa9af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,11 +32,11 @@ defaults: # frees its runner from the org's shared 60-runner pool. Each cap follows the # job's slowest successful leg on 2026-11-09, rounded up to a multiple of 5 # minutes: 1.5 times it for engine-free (109 minutes) and engine-us (73), twice -# for integration-uk (8), and for each shorter job at least 5 minutes and at -# least twice its slowest run. The engine-uk cap is about 1.6x its expected -# slowest leg with microcosm#931's clone-first H2 test: legs took 31-46 minutes -# on main on 2026-11-11, and that test's module runs about 2.6 times as long as -# the H2 parity test (10 minutes in this job), so about 27 minutes more. +# for engine-uk (45) and integration-uk (8), and for each shorter job at least +# 5 minutes and at least twice its slowest run. The engine-uk-after-merge cap +# is about 1.6x its expected leg: microcosm#931's clone-first H2 test runs +# about 2.6 times as long as the H2 parity test (10 minutes in engine-uk), so +# about 27 minutes, plus the sync. # Every job names its runner image. `ubuntu-latest` is whichever image GitHub # assigns: on 2026-11-10 it gave one engine-free job ubuntu-26.04 and another @@ -136,7 +136,7 @@ jobs: needs: select-countries if: needs['select-countries'].outputs.run_uk == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 120 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -151,6 +151,26 @@ jobs: - name: Run UK engine tests run: bash tools/run_engine_uk_tests.sh + engine-uk-after-merge: + # UK engine tests too slow for every pull request run on pushes to main + # only, after merge (microcosm#931). + if: github.event_name == 'push' + runs-on: ubuntu-24.04 + timeout-minutes: 50 + strategy: + fail-fast: false + matrix: + python-version: ["3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Sync workspace with the UK engine + run: uv sync --all-packages --locked --extra uk + - name: Run UK after-merge engine tests + run: bash tools/run_engine_uk_tests.sh engine-uk-after-merge + integration-uk: needs: select-countries if: needs['select-countries'].outputs.run_uk == 'true' diff --git a/changelog.d/931-engine-uk-after-merge-ci.added.md b/changelog.d/931-engine-uk-after-merge-ci.added.md new file mode 100644 index 0000000000..4da316a3cd --- /dev/null +++ b/changelog.d/931-engine-uk-after-merge-ci.added.md @@ -0,0 +1 @@ +A new CI job, `engine-uk-after-merge`, runs on pushes to main only, after merge (#931). It runs the new `engine_after_merge/uk/` test category, UK engine tests too slow for every pull request. Its first module is the clone-first H2 test, which runs the whole UK spine at one and at four geographic clones through the engine, about 27 minutes per leg. `tools/run_engine_uk_tests.sh` takes the job whose files it runs, `engine-uk` by default. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 181ae3f60e..c71a2c4a09 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -268,6 +268,9 @@ of these directories below its package's `tests/` directory: aging, fiscal refresh, release preparation, scoring, or another complete data workflow. Every module in this category runs in a fresh pytest process. - `engine/uk/`: imports or executes the UK engine. +- `engine_after_merge/uk/`: imports or executes the UK engine but is too slow for + every pull request; runs through the `engine-uk-after-merge` job on pushes to + main only, after merge. - `integration/uk/`: runs serially through the `integration-uk` job with the explicit `--run-integration` option. - `installed_emitter/shared/`: starts the separately installed diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ci_integration_job.py b/packages/microcosm-build/tests/engine_free/shared/test_ci_integration_job.py index aa6b851f7d..683ea426dd 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ci_integration_job.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ci_integration_job.py @@ -60,6 +60,22 @@ def test_tests_workflow_selects_every_test_environment() -> None: assert "reporting_groups_for_job" in engine_runner +def test_only_the_after_merge_uk_job_waits_for_a_push_to_main() -> None: + """UK engine tests too slow for every pull request run after merge only.""" + jobs = yaml.safe_load(_TEST_WORKFLOW.read_text(encoding="utf-8"))["jobs"] + after_merge = jobs["engine-uk-after-merge"] + + assert after_merge["if"] == "github.event_name == 'push'" + assert "needs" not in after_merge + assert after_merge["steps"][-1]["run"] == ( + "bash tools/run_engine_uk_tests.sh engine-uk-after-merge" + ) + assert jobs["engine-uk"]["steps"][-1]["run"] == "bash tools/run_engine_uk_tests.sh" + assert [ + name for name, job in jobs.items() if "event_name" in str(job.get("if", "")) + ] == ["engine-uk-after-merge"] + + def test_integration_job_is_uk_selected_and_read_only() -> None: workflow = _TEST_WORKFLOW.read_text(encoding="utf-8") integration = workflow.split("\n integration-uk:\n", maxsplit=1)[1].split( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py b/packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py index f031f5d030..2e65a2b012 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_ci_test_plan.py @@ -86,6 +86,14 @@ "policyengine_uk", False, ), + ( + "engine_after_merge/uk/test_one.py", + "engine-after-merge-uk", + "engine-uk-after-merge", + "uk", + "policyengine_uk", + False, + ), ( "integration/uk/test_one.py", "integration-uk", diff --git a/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py b/packages/microcosm-graph/tests/engine_after_merge/uk/test_h2_clone_first.py similarity index 99% rename from packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py rename to packages/microcosm-graph/tests/engine_after_merge/uk/test_h2_clone_first.py index bf7a24e9bf..efc6221694 100644 --- a/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_clone_first.py +++ b/packages/microcosm-graph/tests/engine_after_merge/uk/test_h2_clone_first.py @@ -9,6 +9,9 @@ clone-index invariant and each stage's outputs as weighted shares, and keeps the populations the exact checks compare. +Too slow for every pull request, the module runs after merge only, in the +engine-uk-after-merge job on pushes to main. + - **Every stage runs**, and at K = 4 every engine call from geographic_expand on runs four simulations, one per clone, in as many calls per stage as at one clone: no stage calls the engine per clone. private_school_attendance diff --git a/tools/ci_test_plan.py b/tools/ci_test_plan.py index 5a93e2c932..6d6c6e3084 100644 --- a/tools/ci_test_plan.py +++ b/tools/ci_test_plan.py @@ -65,6 +65,14 @@ class TestGroup: "engine-uk": TestGroup( ("engine", "uk"), "engine-uk", "uk", engine_module="policyengine_uk" ), + # UK engine tests too slow for every pull request: the engine-uk-after-merge + # job runs them on pushes to main only, after merge (microcosm#931). + "engine-after-merge-uk": TestGroup( + ("engine_after_merge", "uk"), + "engine-uk-after-merge", + "uk", + engine_module="policyengine_uk", + ), "integration-uk": TestGroup( ("integration", "uk"), "integration-uk", diff --git a/tools/run_engine_uk_tests.sh b/tools/run_engine_uk_tests.sh index d072b48496..a74d31c773 100755 --- a/tools/run_engine_uk_tests.sh +++ b/tools/run_engine_uk_tests.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash set -euo pipefail +# The CI job whose files run: engine-uk, or engine-uk-after-merge. +job="${1:-engine-uk}" files=() while IFS= read -r file; do files+=("$file"); done < <( - uv run --no-sync python tools/ci_test_plan.py list-job engine-uk + uv run --no-sync python tools/ci_test_plan.py list-job "$job" ) # Keep country-engine execution serial so each process stays within its From 5034f9b5f129048ffd3c4b356247f53ee959e68e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:03:42 +0000 Subject: [PATCH 068/108] Give each engine simulation rank ids where policyengine-core's 32-bit store would repeat one (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit policyengine-core stores policyengine-uk's five id variables as int32, converting larger ids with numpy's astype without an error, but it builds the entities and their memberships from the int64 columns it is given. Two ids that wrap to one value therefore merge no entity; they become one stored key for every formula that reads the id (policyengine-uk hashes its Warm Home Discount, Universal Credit deduction and birth-month draws on them), and one entity for anything rebuilt from the stored values. microcosm.frame.adapters.engine_identifiers (new; shared, for Max's review) gives a simulation the frame's tables unchanged where no entity's ids repeat once converted to the engine's id type. Where an entity's would, its ids become 1 plus their rank among that simulation's ids, and each person's membership of it becomes the new id of the group it names, never the 0-based rank. Ranks keep id order, so the engine holds the rows in the frame's order and results return as before, by position or by the frame's own ids. The frame is never changed. The module also names the entities whose ids the conversion changes, refuses a request for such an entity's id or membership variable (reading no table unless one of those is requested), and holds a declaration to an installed engine's types. Its rank step refuses to renumber an entity that a declared link's table among its tables names, since it maps only the person table's memberships. It reads a schema and entity tables, never a Frame, and names no country or country module; its refusals name entities, links, columns, variables and types, never an id or a count. national_frame's release rule keeps its own ranker (decision 7): moving it onto rank_identifier_tables, and deriving UK_ENGINE_IDENTIFIER_MAX from the declaration, are left for Max, as the comment on the UK declaration records. The UK adapter declares int32 (UK_ENGINE_IDENTIFIERS) and checks it against the installed engine once per adapter, so building a dataset now builds the adapter's CountryTaxBenefitSystem first; a test holds national_frame's release range to the same width. Its new _dataset_tables applies the rule to engine_tables' tables, and both construction sites build through it, so a call run clone by clone ranks each batch within its own rows. materialize refuses, before any engine is built, a request for an id or membership variable whose entity holds an id the engine cannot store exactly, and reads the frame only for such a request. The adapter builds its simulations with policyengine-uk's EntityIdWarning filtered, since it now checks the stored ids itself, and refuses to build one if policyengine-uk no longer defines that class at utils.entity_ids. graph_kernels adds the module to _ENGINE_COMPUTE_MODULES, which every stage that calls the engine binds, since the walk from the engine the builds inject reaches it. Tests, on invented ids only: - engine_free/shared/test_engine_identifiers.py (new; its layouts are in test_support/microcosm_frame/engine_identifier_frames.py): pass-through as the same table objects, inside int32 and on copy-layout ids past 2**31; a stored repeat renumbered by rank, with 1-based memberships; a seeded sweep under int8 and int16 declarations over every NumPy and nullable integer id dtype; inexact entities and request refusals; the rank step's refusals, a link table naming an entity to renumber among them; and refusals checked by exact string to quote no id, wrapped value or row count. - The adapter, engine-free: the declaration, its once-per-adapter check, and the dataset builder holding it before it builds anything; on ids past 2**31 that stay apart, _dataset_tables hands on the very tables engine_tables made; each batch ranks its own rows, directly and through materialize's batched site and the dataset builder; the request refusals; and the warning filter, which fails closed at both construction sites. test_uk_graph's binding parametrization gains the module, and test_uk_release_identifiers holds the two widths equal. - engine/uk/test_policyengine_uk_identifiers.py (new): the declaration is the installed engine's; on copy-layout ids every value, the id-keyed draws included, is a direct Microsimulation's at 2024, at 2025 and at 2025 projected from 2024; households 7 and 7 + 2**32 take different Warm Home Discount draws, and every value equals the same call over the frame renumbered by hand, at the dataset's year, projected through policyengine-uk's council-tax and student-loan joins, and clone by clone; and the warning is filtered at both construction sites while a direct Microsimulation still records it. - The H2 parity test records, as two booleans, that its simulations reach the rule and that none renumbers, and asserts both after the parity check, with no second run. Text: docs/uk-full-build-graph.md states the rule and the binding; the engine-tier adapter test's comment and the clone-first H2 test's docstring say what now holds; and changelog.d/931-engine-simulation-rank-ids.fixed.md is new. What moves. On a frame whose stored ids do not repeat the engine's input is unchanged byte for byte, so no value moves. Where an entity's stored ids repeat, only that simulation's values keyed on its ids move: for households, the Warm Home Discount draw, which runs for 2025 to 2030 and adds only to HBAI income, which spine stages request only at 2024. A batched call's id-keyed values can differ from one simulation's over the whole frame wherever either renumbers. An id request for an entity the engine cannot store exactly is now refused; no stage makes one. The adapter's simulations no longer write EntityIdWarning to stderr. Every UK graph node re-keys. All 59 spine kernels move, since they hash the adapter or graph_kernels, and the 16 that call the engine now also bind engine_identifiers, which no other kernel hashes. So do the 16 full-build and national kernels that hash the adapter: the bound-spine root, seven dense-population kernels, the measures, the national problem, readback and gates, the export readback, both full gate batteries and the rotated holdout. The other 26 UK kernels hash none of the changed modules and re-key through their population's version. No gate phase code moves. No other country's keys move: no kernel hash or node key changes in the transport, entitlement or NZ toy graphs or the US post-transfer graph, and the AM, BE, NZ and US spec identities are unchanged. Outside the keys, the US annual static-aging manifest's runtime source_sha256, which hashes every microcosm.frame module, gains the new module and changes the adapter's entry. No declaration, generated file, gates.json, schema, receipt or sidecar field changes. Checks. Engine-free: test_engine_identifiers.py (21 passed), the engine-free test_policyengine_uk_adapter.py (63), test_uk_release_identifiers.py (10), test_uk_graph.py in full (60) and test_uk_rules_engine_provenance.py (15); ruff check and ruff format --check on every edited file, and ruff check .; ci_test_plan.py verify; graph_acceptance_burndown.py --verify. Through the engine slot, one job at a time: the new engine-tier file with test_policyengine_uk_batches.py and the engine-tier test_policyengine_uk_adapter.py (34 passed); the H2 parity test with its record (1 passed); and the clone-first K = 4 module (13 passed). On the synthetic H2 fixture the legacy oracle's final frame (every table, column and dtype, the weights, the strata and the mass log), its stage records and every stage's evidence are identical at the parent and here, and the fixture regenerates byte-identically. On eight invented frames whose stored ids do not repeat, this tree hands UKSingleYearDataset the parent's inputs byte for byte, as the very tables engine_tables made; a ninth, of float64 ids past 2**31, renumbers here, since this machine's numpy converts them to one stored value. At the parent, with the new module, its test layouts and the new engine-tier file overlaid, the three collision tests and both warning tests fail, the declaration test fails for want of the check, and the three end-to-end cases pass; a 0-based membership map fails the two unbatched collision tests (checked in review). The new engine-free checks fail on the parity review's surviving mutants: _dataset_tables returning copies, _build_dataset skipping the declaration or handing the engine engine_tables' own tables, and a batched call ranking the whole call's frame. Decision 12, as María ruled on 11 Nov. Co-Authored-By: Claude Opus 5.5 --- .../931-engine-simulation-rank-ids.fixed.md | 14 + docs/uk-full-build-graph.md | 4 +- .../build/uk_runtime/graph_kernels.py | 16 +- .../tests/engine_free/uk/test_uk_graph.py | 2 + .../uk/test_uk_release_identifiers.py | 15 + .../frame/adapters/engine_identifiers.py | 585 ++++++++++++++++ .../frame/adapters/policyengine_uk.py | 139 +++- .../engine/uk/test_policyengine_uk_adapter.py | 6 +- .../uk/test_policyengine_uk_identifiers.py | 653 ++++++++++++++++++ .../shared/test_engine_identifiers.py | 547 +++++++++++++++ .../uk/test_policyengine_uk_adapter.py | 439 ++++++++++++ .../engine/uk/test_acceptance_h_parity.py | 35 +- .../uk/test_h2_clone_first.py | 13 +- .../engine_identifier_frames.py | 604 ++++++++++++++++ 14 files changed, 3052 insertions(+), 20 deletions(-) create mode 100644 changelog.d/931-engine-simulation-rank-ids.fixed.md create mode 100644 packages/microcosm-frame/src/microcosm/frame/adapters/engine_identifiers.py create mode 100644 packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_identifiers.py create mode 100644 packages/microcosm-frame/tests/engine_free/shared/test_engine_identifiers.py create mode 100644 test_support/microcosm_frame/engine_identifier_frames.py diff --git a/changelog.d/931-engine-simulation-rank-ids.fixed.md b/changelog.d/931-engine-simulation-rank-ids.fixed.md new file mode 100644 index 0000000000..09e228d840 --- /dev/null +++ b/changelog.d/931-engine-simulation-rank-ids.fixed.md @@ -0,0 +1,14 @@ +Each policyengine-uk simulation the UK adapter builds now holds ids that policyengine-core's 32-bit store keeps apart (#931; decision 12, María's rulings of 11 Nov). policyengine-core stores policyengine-uk's five id variables (`person_id`, `benunit_id`, `household_id`, `person_benunit_id` and `person_household_id`) as int32, converting them with numpy's `astype`, which wraps an id outside the type without an error. It builds the entities and memberships from the int64 columns it is given, so two ids that wrap to one value merge no entity: they become one stored key for every formula that reads the id. policyengine-uk 2.174.1 hashes its Warm Home Discount draw on `household_id`, its two Universal Credit deduction draws on `benunit_id` and its birth-month spread on `person_id`. Anything rebuilt from the stored values would see one entity. Within one clone's batch every id carries the same clone offset, so a repeat there needs a copy-layer ordinal on at least one of the two ids. + +- **The rule** (`microcosm.frame.adapters.engine_identifiers`, shared, for Max's review). `EngineIdentifiers(stored_dtype)` declares the integer type an engine stores a schema's id and membership columns as. `engine_identifier_tables(schema, tables, identifiers)` hands a simulation the tables it is given when no entity's ids repeat once converted as the engine converts them (`stored_identifier_repeats`). An entity whose stored ids would repeat takes 1 plus each id's rank among that simulation's ids, in id order, and each person's membership of it takes the new id of the group it names, never the 0-based rank (`rank_identifier_tables`, the step that ranks and maps and decides nothing). Ranks are distinct, fit the type and keep id order, so the engine holds every entity's rows in the frame's order, and results still return by position or by the frame's own ids. The frame is never changed. The module also names the entities whose ids the conversion changes (`inexact_identifier_entities`), refuses a request for such an entity's id or membership variable (`refuse_inexact_identifier_requests`, which reads no table unless an id or membership variable is requested), and holds a declaration to an installed engine's types (`require_stored_dtypes`). It reads a schema and entity tables, never a Frame, and names no country. It refuses to renumber an entity that a declared link's table among the tables names, since it maps only the person table's memberships. Its refusals name entities, links, columns, variables and types, never an id or a count. national_frame's release rule keeps its own ranker for now; moving it onto `rank_identifier_tables`, and deriving `UK_ENGINE_IDENTIFIER_MAX` from the declaration, are owed (decision 7). +- **The UK adapter.** `UK_ENGINE_IDENTIFIERS` declares int32, and `_require_identifier_dtypes` checks it against the installed engine's five id variables once per adapter; a test holds national_frame's release range (`UK_ENGINE_IDENTIFIER_MAX`) to the same width. `_dataset_tables` applies the rule to `engine_tables`' tables, and both construction sites build through it, so a call run clone by clone ranks each batch within its own rows. `materialize` refuses, before any engine is built, a request for an id or membership variable whose entity holds an id the engine cannot store exactly, since the engine would return wrapped ids or ranks for it. Each simulation is built with policyengine-uk's `EntityIdWarning` filtered, since the adapter now checks the stored ids itself, and the adapter refuses to build one if policyengine-uk no longer defines that class at `utils.entity_ids`. Building a dataset now builds the adapter's `CountryTaxBenefitSystem` first, which is inert. The stages that call the engine bind the new module beside `microcosm.frame.materialize` (`_ENGINE_COMPUTE_MODULES`), as the walk from the engine the builds inject reaches it. +- **What moves.** + - Nothing on a frame whose stored ids do not repeat: `UKSingleYearDataset` is built from the same tables, byte for byte, so every engine output is the same. + - Elsewhere, only that simulation's values keyed on the renumbered entity's ids. For households that is the Warm Home Discount draw, which runs for 2025 to 2030 and adds only to HBAI income, which spine stages request only at 2024. A batched call's id-keyed values can differ from one simulation's over the whole frame wherever either renumbers, since each batch ranks its own rows. + - The adapter's simulations no longer write `EntityIdWarning` to stderr, and an id request for an entity the engine cannot store exactly is refused; no stage makes one. + - Every UK graph node re-keys: the kernels that hash the adapter directly, and every other node through its population's version. No other country's keys move. The US annual static-aging manifest's `source_sha256`, which hashes every module under `microcosm.frame`, gains the new file and changes the adapter's entry; it is provenance, not a node key. No declaration, generated file, `gates.json`, schema, receipt or sidecar field changes. +- **Tests** (invented data only). + - The shared module (`tests/engine_free/shared/test_engine_identifiers.py`): pass-through, the same table objects, on ids inside int32 and on copy-layout ids past 2**31 that convert out of int64 order; a stored repeat renumbered by rank, with 1-based memberships; a seeded sweep of a few hundred frames under int8 and int16 declarations, over NumPy and nullable, signed and unsigned id dtypes, against the tests' own reading of the conversion; inexact entities and request refusals; the rank step's refusals, a link table naming an entity to renumber among them; and refusals checked by exact string to quote no id, wrapped value or row count. + - The adapter, engine-free: the declaration, its once-per-adapter check, and the dataset builder holding it before it builds anything; on ids past 2**31 that stay apart, `_dataset_tables` hands on the very tables `engine_tables` made; each batch ranks its own rows, through `materialize`'s batched site and the dataset builder too; the request refusals; and the warning filter fails closed at both construction sites. The binding walk gains the module, and the two widths agree. + - Through the engine (`tests/engine/uk/test_policyengine_uk_identifiers.py`): the declaration is the installed engine's. On copy-layout ids every value, the id-keyed draws included, is a direct Microsimulation's over the frame's engine tables at 2024, at 2025 and at 2025 projected from 2024, with each household's member count and an echoed household input in row order. Households 7 and 7 + 2**32 take different Warm Home Discount draws, and every value is the same call's over the frame renumbered by hand: at the dataset's year, projected through policyengine-uk's council-tax and student-loan joins on the raw memberships, and clone by clone at two clones. These three fail where the adapter passes the frame's ids through unchanged, and a 0-based membership map fails the first two. The warning is filtered at both construction sites, while a direct Microsimulation over the same tables still records it. + - The H2 parity test records, as two booleans, that its simulations reach the rule and that none of them renumbers. diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index d698788d79..4b3c0b8bbe 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -112,9 +112,9 @@ The full target compiler preserves the unreduced band-edge register, reference-p `--engine-blocks K` resolves `uk.full.measures` with one policyengine-uk engine per clone rather than one over the pool; above one, it must equal `--n-clones`. Each block's household weights are scaled to the pool's mass. In every simulation the block builds (the measure simulation, its baseline, and the annual-income and counterfactual simulations with theirs), `shareholding`, `corporate_land_value` and `consumption_shareholding` are set, at each period the block computes, to the block's own value times `c_b = f_b·S_b/S`. That is the block's pool-mass factor times its allocation-key total over the pool's, both from totals the representation records (microcosm#931). Before it sets them, the block checks that the engine's own key total is `f_b·S_b` and that the engine carries each key and the weights by one factor per year. The annual-income simulation carries other spells by the engine's own growth parameters, so every block uses the same factor. A guard records every formula each simulation runs once the shares are set, and judges policyengine-uk's whole-population formulas against the UK declaration (`microcosm.frame.adapters.policyengine_uk_population`, on the shared `engine_population` machinery). The three share formulas must be the corrected inputs at every period, and `months_since_last_birthday` and `attends_private_school` stored inputs. The engine's income and wealth deciles and relative-poverty flags are refused, and any other such formula that runs must be zero under the parameters checked. Any work in the Universal Credit rebalancing years (2026 on) is refused too, since policyengine-uk draws new-claimant status there by benefit-unit position. The measure step records a violation rather than raising it. Once the blocks ran, `engine_population_representation.exact` holds only if every block applied its correction, passed its checks and got a clean verdict, and `exact_basis` (`pool_share_injection_with_population_guard`) names that basis. The measures then equal one engine over the pool to float32 rounding, not bitwise. An inexact per-block run is not releasable. So while policyengine-uk draws new-claimant status by position, a calibration year, or a dated measure's year, of 2026 or later leaves every per-clone run inexact, and so not releasable, and refuses every batched spine call. A single block, as on the national line, is exact by construction and records no correction or guard fields. -A spine engine call whose frame carries more than one clone runs one simulation per clone, since one simulation over a cloned spine does not fit in memory. `PolicyEngineUKEngine.materialize` decides from the frame, not from a setting: a call whose household table carries `household_clone_index` with more than one clone runs its simulations in ascending clone order, and each is released before the next is built. A frame without the column, or with one clone, takes one simulation as before. A batched call's requests are judged first. A period in the rebalancing years, a refused formula, a frame without `months_since_last_birthday`, or a request for `attends_private_school`, `private_school_vat`, `baseline_private_school_vat` or `gov_tax` before stage 52 stores attendance refuses the call before any engine is built. Until then the spine's pass lets attendance run only where the three parameters the private-school VAT rate reads equal their baseline's and `private_school_vat_change` comes out zero, and only if `gov_tax` does not run. Unlike a measure block, each batch reads the pool's shares at the frame's own weights, unscaled (`c_b = S_b/S`), and is judged under the spine's pass, which raises on any violation rather than recording it, since the values come back as plain arrays. People come back in the frame's order and groups in ascending id order, as one simulation returns them. Every spine frame from `geographic_expand` on carries `household_clone_index`, 0 throughout while the spine build runs one clone, so every spine call takes one simulation today, and batching moves no spine value. +A spine engine call whose frame carries more than one clone runs one simulation per clone, since one simulation over a cloned spine does not fit in memory. `PolicyEngineUKEngine.materialize` decides from the frame, not from a setting: a call whose household table carries `household_clone_index` with more than one clone runs its simulations in ascending clone order, and each is released before the next is built. A frame without the column, or with one clone, takes one simulation as before. A batched call's requests are judged first. A period in the rebalancing years, a refused formula, a frame without `months_since_last_birthday`, or a request for `attends_private_school`, `private_school_vat`, `baseline_private_school_vat` or `gov_tax` before stage 52 stores attendance refuses the call before any engine is built. Until then the spine's pass lets attendance run only where the three parameters the private-school VAT rate reads equal their baseline's and `private_school_vat_change` comes out zero, and only if `gov_tax` does not run. Unlike a measure block, each batch reads the pool's shares at the frame's own weights, unscaled (`c_b = S_b/S`), and is judged under the spine's pass, which raises on any violation rather than recording it, since the values come back as plain arrays. People come back in the frame's order and groups in ascending id order, as one simulation returns them. Each simulation holds the frame's ids where policyengine-core's 32-bit store keeps them apart; an entity whose stored ids would repeat takes 1 plus each id's rank in that simulation's frame, and each person's membership takes the new id of the group it names, so rows keep their order and the frame keeps its ids. Every spine frame from `geographic_expand` on carries `household_clone_index`, 0 throughout while the spine build runs one clone, so every spine call takes one simulation today, and batching moves no spine value. -policyengine-uk's `attends_private_school` ranks each household's market income plus benefits, less its income tax and National Insurance, over the people a simulation holds, so stage 52 (`private_school_attendance`) restates the formula's last step over the whole frame. It computes that income from household-local engine values, ranks it with microdf's `MicroSeries.percentile_rank`, and reads the attendance rates and population factor from the engine's own parameters. Over one simulation it also asks for the engine's flags, stores them once the restatement matches them bit for bit, and raises otherwise. Clone by clone it asks only for the household-local values and stores the restatement. Its evidence records `rank_route` (`engine_with_parity` or `pooled_restatement`) and `engine_parity_checked`. The spine sidecar's `rules_engine` gains `engine_batches`, and beside it `microdf_python`, the microdf-python version whose weighted percentile rank `private_school_attendance` then stores, only on a spine whose calls run clone by clone, and the engine identity does not move with batching. Every stage that calls the engine through the adapter binds the adapter, `engine_population` and `policyengine_uk_population` into its implementation hash, and `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. +policyengine-uk's `attends_private_school` ranks each household's market income plus benefits, less its income tax and National Insurance, over the people a simulation holds, so stage 52 (`private_school_attendance`) restates the formula's last step over the whole frame. It computes that income from household-local engine values, ranks it with microdf's `MicroSeries.percentile_rank`, and reads the attendance rates and population factor from the engine's own parameters. Over one simulation it also asks for the engine's flags, stores them once the restatement matches them bit for bit, and raises otherwise. Clone by clone it asks only for the household-local values and stores the restatement. Its evidence records `rank_route` (`engine_with_parity` or `pooled_restatement`) and `engine_parity_checked`. The spine sidecar's `rules_engine` gains `engine_batches`, and beside it `microdf_python`, the microdf-python version whose weighted percentile rank `private_school_attendance` then stores, only on a spine whose calls run clone by clone, and the engine identity does not move with batching. Every stage that calls the engine through the adapter binds the adapter, `engine_population` and `policyengine_uk_population` into its implementation hash, and `microcosm.frame.materialize` and `microcosm.frame.adapters.engine_identifiers`, through which the adapter builds each simulation's dataset and the ids it holds. ## Replay and outputs diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index 21a1aafe5f..efcb54c18d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -35,7 +35,7 @@ from microcosm.frame import Frame, MassChangeRecord, WeightKind from microcosm.frame import materialize as frame_materialize from microcosm.frame import rules as frame_rules -from microcosm.frame.adapters import engine_population +from microcosm.frame.adapters import engine_identifiers, engine_population from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from microcosm.frame.adapters import ( policyengine_uk_population as uk_engine_population_declaration, @@ -229,12 +229,16 @@ ) # What else the injected engine computes through when those stages call it: # the adapter builds each simulation's dataset through frame.materialize's -# engine_tables. test_uk_graph derives the group from a second walk, rooted at -# the PolicyEngineUKEngine the builds inject and followed from the methods the +# engine_tables, and the ids that simulation holds through engine_identifiers. +# test_uk_graph derives the group from a second walk, rooted at the +# PolicyEngineUKEngine the builds inject and followed from the methods the # engine stages call on it (microcosm#931). It is kept apart from -# _ENGINE_MODULES, which no other stage binds, because the SPI spine's three -# stages and the band donors bind frame.materialize for their own tables. -_ENGINE_COMPUTE_MODULES = (frame_materialize,) +# _ENGINE_MODULES, which no other stage binds, because another stage may bind +# one of its modules for its own tables, which test_uk_graph allows only where +# the stage's own transform computes through it: the SPI spine's three stages +# and the band donors bind frame.materialize. No stage reaches +# engine_identifiers except through the engine. +_ENGINE_COMPUTE_MODULES = (frame_materialize, engine_identifiers) # What every stage that calls the engine binds. _ENGINE_STAGE_MODULES = (*_ENGINE_MODULES, *_ENGINE_COMPUTE_MODULES) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py index 89d9ab9f15..4853b1cf5e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph.py @@ -992,6 +992,8 @@ def test_the_engine_call_scan_sees_any_receiver(tmp_path) -> None: ("microcosm.frame.adapters.policyengine_uk_population", False), # The adapter builds each simulation's dataset through it. ("microcosm.frame.materialize", False), + # And the ids each simulation holds (microcosm#931). + ("microcosm.frame.adapters.engine_identifiers", False), ], ) def test_engine_module_changes_invalidate_every_stage_that_calls_the_engine( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_identifiers.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_identifiers.py index 952e8ee718..642a0b3f52 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_release_identifiers.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_release_identifiers.py @@ -36,6 +36,8 @@ ) from microcosm.build.uk_runtime.weight_year import uk_dataset_year_frame from microcosm.frame import WeightKind +from microcosm.frame.adapters.engine_identifiers import identifier_columns +from microcosm.frame.adapters.policyengine_uk import UK_ENGINE_IDENTIFIERS, UK_SCHEMA from test_support.microcosm_build.uk_graph_terminal import _frame as _ladder_frame # Two copies of a three-household survey, numbered the way the clone stages @@ -187,6 +189,19 @@ def test_range_failures_name_the_column_only() -> None: assert uk_identifier_range_failures(edge) == [f"Written 'household_id' {above}"] +def test_the_release_range_is_the_engine_adapters_declared_width() -> None: + # The release rule keeps its own ranker and range for the released file, + # and the engine adapter declares how policyengine-core stores the same + # columns in each simulation (microcosm#931). Until the release rule moves + # onto the shared rank step, the two widths are held equal here, and the + # range check covers the columns the declaration covers. + assert np.iinfo(UK_ENGINE_IDENTIFIERS.stored_dtype).max == UK_ENGINE_IDENTIFIER_MAX + above = "has a value above the 32-bit range policyengine-uk stores." + assert set(uk_identifier_range_failures(_tables())) == { + f"Written {column!r} {above}" for column in identifier_columns(UK_SCHEMA) + } + + def test_the_release_frame_leaves_renumbered_and_loads_back_to_the_build( tmp_path, ) -> None: diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/engine_identifiers.py b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_identifiers.py new file mode 100644 index 0000000000..9b349b89c7 --- /dev/null +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/engine_identifiers.py @@ -0,0 +1,585 @@ +"""The ids each engine simulation holds for a frame's entities (microcosm#931). + +policyengine-core keeps a dataset's ids in two places. Where a country package +builds a simulation from the tables an adapter hands it, as the package this +rule was first written for does, the entities, and each person's membership of +them, are built from the id and membership columns as given. Core also stores +those columns as the values of the id variables, each converted by numpy's +``astype`` to its variable's type, which wraps an id outside the type without +an error. Two ids that wrap to one stored value therefore merge no entity, but +they become one key for every formula that reads the stored id (a draw hashed +on it, a tie broken on it), and one entity for anything rebuilt from the +stored values. An adapter that adopts this rule first checks how its engine +builds its entities. + +An adapter declares the type with :class:`EngineIdentifiers`, and +:func:`require_stored_dtypes` holds the declaration to the installed engine. +The rule, :func:`engine_identifier_tables`, takes each simulation's own tables: + +- an entity whose ids stay apart once stored keeps them, so a simulation in + which no entity's stored ids would repeat receives the tables it was given, + and its input is unchanged byte for byte; +- an entity whose stored ids would repeat takes 1 plus each id's rank among + the entity's ids in that simulation, in id order, and each person's + membership of it takes the new id of the group it names (1 plus the group's + rank, never the 0-based rank). Ranks are distinct, fit the type and keep id + order, so the engine holds every entity's rows in the frame's order. + +The frame is never changed, and no result is ever mapped back by the engine's +ids: an adapter returns values by position or by the frame's own ids. The +engine holds wrapped ids or ranks for an entity whose ids it cannot store +exactly, so :func:`refuse_inexact_identifier_requests` refuses a request for +that entity's id or membership variable rather than return them. + +:func:`rank_identifier_tables` is the step that ranks and maps, and it decides +nothing. A release rule that renumbers every entity of a released file takes +the same step with its own ranker; moving such a rule onto this function is +owed. + +Every function here that reads ids takes a schema and entity tables (a mapping +from entity to table, or a reader for one), never a Frame, so a caller holding +only a table dict can use it. Refusals name entities, links, columns, +variables and types, never an id or a count. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.frame.schema import EntitySchema + +__all__ = [ + "EngineIdentifierTables", + "EngineIdentifiers", + "engine_identifier_tables", + "identifier_columns", + "inexact_identifier_entities", + "rank_identifier_tables", + "refuse_inexact_identifier_requests", + "require_stored_dtypes", + "stored_identifier_repeats", +] + +_INT64_MAX = int(np.iinfo(np.int64).max) + + +@dataclass(frozen=True) +class EngineIdentifiers: + """How an engine stores a schema's id and membership columns. + + The engine converts every id and membership column to ``stored_dtype`` with + numpy's ``astype``, as policyengine-core converts a stored input: an + integer outside the type wraps and a float inside it is truncated, without + an error; a float outside the type converts as the machine's numpy + converts it (C leaves that cast undefined), so whether such ids repeat once + stored is the machine's, as it is in the engine. + + Attributes: + stored_dtype: The integer type the columns are stored as, normalised by + ``np.dtype``. + + Raises: + TypeError: If ``stored_dtype`` is not a NumPy integer type. + """ + + stored_dtype: np.dtype + + def __post_init__(self) -> None: + try: + dtype = np.dtype(self.stored_dtype) + except TypeError: + dtype = None + if dtype is None or not np.issubdtype(dtype, np.integer): + raise TypeError( + "EngineIdentifiers.stored_dtype must be a NumPy integer type, " + f"got {self.stored_dtype!r}." + ) + object.__setattr__(self, "stored_dtype", dtype) + + +@dataclass(frozen=True) +class EngineIdentifierTables: + """The tables one engine simulation receives, and the entities they renumber. + + Attributes: + tables: Entity -> the table the simulation receives, in the given + tables' order. Where nothing is renumbered these are the given + tables themselves. Otherwise each renumbered entity's table, and + the person table that holds the memberships, are copies, and every + other table is the given object. + renumbered: The entities whose ids are 1 plus their rank, in schema + order; empty where every entity passes its ids through. + """ + + tables: dict[str, pd.DataFrame] + renumbered: tuple[str, ...] + + +def identifier_columns(schema: EntitySchema) -> Mapping[str, str]: + """Each id and membership column of ``schema``, and the entity whose ids it holds. + + The person entity's id column holds the person entity's ids. Each group's + id column (``{group}_id``) and the person table's membership column of it + (``person_{group}_id``) hold the group's. In schema order: the person id, + then each group's id and membership columns. + + Raises: + ValueError: If two of the columns share a name. + """ + + columns = {schema.person_id_column: schema.person_entity} + for group in schema.group_entities: + for column in (schema.id_column(group), schema.membership_column(group)): + if column in columns: + raise ValueError(f"The schema names the id column {column!r} twice.") + columns[column] = group + return MappingProxyType(columns) + + +def stored_identifier_repeats( + schema: EntitySchema, + tables: Mapping[str, pd.DataFrame], + identifiers: EngineIdentifiers, +) -> tuple[str, ...]: + """The entities whose id column repeats a value once the engine stores it. + + Each entity's ids are converted as policyengine-core converts a stored + input, ``np.asarray(column).astype(stored_dtype)``; a float outside the + type converts as the engine's does, without a warning. A repeat is found on + a sorted copy, where equal values sit side by side. Distinct ids that wrap + to one value repeat, and so do ids the tables themselves repeat. + + Returns: + The entities, in schema order: names only. + + Raises: + TypeError: If the engine could not store an entity's ids at all. + ValueError: If a table or an id column is missing. + """ + + stored_dtype = _stored_dtype(identifiers) + _require_tables(tables) + repeats = [] + for entity in schema.entities: + column, values = _entity_ids(schema, tables, entity) + stored = _stored(values, stored_dtype, entity=entity, column=column) + ordered = np.sort(stored) + if bool((ordered[1:] == ordered[:-1]).any()): + repeats.append(entity) + return tuple(repeats) + + +def inexact_identifier_entities( + schema: EntitySchema, + tables: Mapping[str, pd.DataFrame], + identifiers: EngineIdentifiers, +) -> tuple[str, ...]: + """The entities with an id the engine's conversion changes. + + An id is changed when it lies outside ``stored_dtype`` or is not a whole + number, and the engine then holds another value for it. Distinct whole + numbers inside the type convert to themselves, so every entity + :func:`stored_identifier_repeats` names is one here, unless the tables + themselves repeat its ids. + + Returns: + The entities, in schema order: names only. + + Raises: + TypeError: If the engine could not store an entity's ids at all. + ValueError: If a table or an id column is missing. + """ + + stored_dtype = _stored_dtype(identifiers) + _require_tables(tables) + inexact = [] + for entity in schema.entities: + column, values = _entity_ids(schema, tables, entity) + if not _exact(values, stored_dtype, entity=entity, column=column): + inexact.append(entity) + return tuple(inexact) + + +def rank_identifier_tables( + schema: EntitySchema, + tables: Mapping[str, pd.DataFrame], + entities: Iterable[str], +) -> dict[str, pd.DataFrame]: + """``tables`` with each named entity's ids replaced by 1 plus their rank. + + The step that ranks and maps. It decides nothing: + :func:`engine_identifier_tables` chooses the entities. Each named entity's + id column becomes 1 plus each id's rank among the entity's ids, ascending, + as int64, so a group table sorted by id is numbered 1 to n in row order. + For a named group, the person table's membership column becomes the engine + id of the group each person names, 1 plus that group's rank, found by + ``searchsorted`` on the sorted ids. Never the 0-based rank. Rows keep their + order and index, and every other column is unchanged. A declared link's + table is never mapped, so one among ``tables`` that names an entity to + renumber is refused rather than handed back holding the old ids. + + Args: + schema: The entity structure. + tables: Entity -> table. Nothing is written into them. + entities: The entities to renumber, any declared ones in any order. + + Returns: + Entity -> table, in the order of ``tables``: copies of the named + entities' tables and, when a group is named, of the person table. Every + other table is the given object. + + Raises: + TypeError: If ``entities`` is one name rather than a collection. + ValueError: If an entity is not declared; if ``tables`` hold a + declared link's table that names an entity to renumber; or if a + named entity's ids are missing or repeat, or a membership names no + group of a named entity, which a Frame rules out. + """ + + _require_tables(tables) + if isinstance(entities, str): + raise TypeError("entities must be a collection of entity names, not one.") + named = set(entities) + undeclared = sorted(named - set(schema.entities)) + if undeclared: + raise ValueError( + f"Cannot rank the undeclared entities {undeclared}; the schema " + f"declares {list(schema.entities)}." + ) + linked = [ + link + for link in schema.links + if link.name in tables and named & {link.left_entity, link.right_entity} + ] + if linked: + sides = { + side for link in linked for side in (link.left_entity, link.right_entity) + } + raise ValueError( + f"Cannot renumber {sorted(named & sides)}: the tables hold the link " + f"tables {[link.name for link in linked]}, whose ids this step does " + "not map." + ) + result = dict(tables) + person_entity = schema.person_entity + person_columns: dict[str, np.ndarray] = {} + for entity in (entity for entity in schema.entities if entity in named): + table = _entity_table(tables, entity) + column = schema.entity_id_column(entity) + ranks, ordered = _ranks(table, entity, column) + if entity == person_entity: + person_columns[column] = ranks + continue + renumbered = table.copy() + renumbered[column] = ranks + result[entity] = renumbered + membership = schema.membership_column(entity) + person = _entity_table(tables, person_entity) + members = _complete_values(person, person_entity, membership) + person_columns[membership] = 1 + _positions( + ordered, + members, + group=entity, + person_entity=person_entity, + column=membership, + ) + if person_columns: + person = _entity_table(tables, person_entity).copy() + for column, values in person_columns.items(): + person[column] = values + result[person_entity] = person + return result + + +def engine_identifier_tables( + schema: EntitySchema, + tables: Mapping[str, pd.DataFrame], + identifiers: EngineIdentifiers, +) -> EngineIdentifierTables: + """The tables one simulation receives: the engine's rule over its own tables. + + Pass ``tables`` as the engine would receive them (an adapter's engine + copies of the frame's tables). An entity whose ids would repeat once stored + (:func:`stored_identifier_repeats`) is renumbered by + :func:`rank_identifier_tables`; every other entity passes its ids through. + When nothing repeats, the result holds the given tables themselves. + + Raises: + TypeError: If the engine could not store an entity's ids at all. + ValueError: If a table or an id column is missing, or an entity to + renumber has more rows than the stored type can number from 1 or + is named by a declared link whose table ``tables`` hold. + RuntimeError: If any entity's engine ids still repeat once stored, so + the rank step no longer keeps them apart. + """ + + stored_dtype = _stored_dtype(identifiers) + repeats = stored_identifier_repeats(schema, tables, identifiers) + if not repeats: + return EngineIdentifierTables(tables=dict(tables), renumbered=()) + capacity = int(np.iinfo(stored_dtype).max) + crowded = [entity for entity in repeats if len(tables[entity]) > capacity] + if crowded: + raise ValueError( + f"The ids of {crowded} repeat once the engine stores them as " + f"{stored_dtype}, and those entities have more rows than " + f"{stored_dtype} can number from 1, so their ranks would not fit it " + "either." + ) + ranked = rank_identifier_tables(schema, tables, repeats) + still = stored_identifier_repeats(schema, ranked, identifiers) + if still: + raise RuntimeError( + f"After renumbering {list(repeats)}, the engine ids of {list(still)} " + f"still repeat once stored as {stored_dtype}." + ) + return EngineIdentifierTables(tables=ranked, renumbered=repeats) + + +def refuse_inexact_identifier_requests( + schema: EntitySchema, + variables: Iterable[str], + identifiers: EngineIdentifiers, + *, + table: Callable[[str], pd.DataFrame], +) -> None: + """Refuse a request for an id the engine cannot return as the frame holds it. + + ``variables`` is first intersected with :func:`identifier_columns`; when no + requested name is an id or membership column, this returns having read no + table. Otherwise ``table(entity)`` is read once for each requested column's + entity, and the request is refused when that entity's ids are inexact + (:func:`inexact_identifier_entities`): the engine holds wrapped ids or + ranks for it, not the frame's. + + Args: + schema: The entity structure. + variables: The requested variable names, iterated once. + identifiers: How the engine stores the id and membership columns. + table: Reads one entity's table, such as ``Frame.table``. + + Raises: + TypeError: If ``variables`` is one name rather than a collection. + ValueError: Naming the refused variables, their entities and the type. + """ + + stored_dtype = _stored_dtype(identifiers) + if isinstance(variables, str): + raise TypeError("variables must be a collection of names, not one name.") + columns = identifier_columns(schema) + requested = [name for name in dict.fromkeys(variables) if name in columns] + if not requested: + return + exact: dict[str, bool] = {} + refused = [] + for name in requested: + entity = columns[name] + if entity not in exact: + column = schema.entity_id_column(entity) + values = _values(table(entity), entity, column) + exact[entity] = _exact(values, stored_dtype, entity=entity, column=column) + if not exact[entity]: + refused.append(name) + if refused: + entities = list(dict.fromkeys(columns[name] for name in refused)) + raise ValueError( + f"The engine stores ids as {stored_dtype} and cannot return " + f"{refused} as the frame holds them: the ids of {entities} are not " + f"all whole numbers inside {stored_dtype}, so the engine holds other " + "values for them." + ) + + +def require_stored_dtypes( + identifiers: EngineIdentifiers, dtypes: Mapping[str, object] +) -> None: + """Refuse an engine that stores an id column as a type other than the declared one. + + Args: + identifiers: The adapter's declaration. + dtypes: Column -> the type the installed engine stores it as (such as a + variable's ``dtype``), normalised by ``np.dtype``. It must hold + every column of ``identifier_columns(schema)``; a column left out + is not checked. + + Raises: + ValueError: Naming each differing column, its type and the declared + type. + """ + + declared = _stored_dtype(identifiers) + differing = [] + for column, given in dtypes.items(): + try: + dtype = np.dtype(given) + except TypeError: + differing.append(f"{column!r} as {given!r}") + continue + if dtype != declared: + differing.append(f"{column!r} as {dtype}") + if differing: + raise ValueError( + f"The installed engine stores {', '.join(differing)}, not as the " + f"declared {declared}; review the adapter's id declaration against " + "the engine." + ) + + +def _stored_dtype(identifiers: object) -> np.dtype: + if not isinstance(identifiers, EngineIdentifiers): + raise TypeError( + "identifiers must be an EngineIdentifiers declaration, got " + f"{type(identifiers).__name__}." + ) + return identifiers.stored_dtype + + +def _require_tables(tables: object) -> None: + if not isinstance(tables, Mapping): + raise TypeError( + f"tables must map each entity to its table, got {type(tables).__name__}." + ) + + +def _entity_table(tables: Mapping[str, pd.DataFrame], entity: str) -> pd.DataFrame: + if entity not in tables: + raise ValueError(f"The tables hold no {entity!r} table.") + return tables[entity] + + +def _values(table: pd.DataFrame, entity: str, column: str) -> np.ndarray: + """A column's values as numpy reads them, which is how the engine reads them.""" + + if column not in table.columns: + raise ValueError(f"The {entity!r} table lacks the column {column!r}.") + return np.asarray(table[column]) + + +def _complete_values(table: pd.DataFrame, entity: str, column: str) -> np.ndarray: + values = _values(table, entity, column) + if bool(table[column].isna().any()): + raise ValueError(f"The {entity!r} column {column!r} has missing values.") + return values + + +def _entity_ids( + schema: EntitySchema, tables: Mapping[str, pd.DataFrame], entity: str +) -> tuple[str, np.ndarray]: + column = schema.entity_id_column(entity) + return column, _values(_entity_table(tables, entity), entity, column) + + +def _stored( + values: np.ndarray, stored_dtype: np.dtype, *, entity: str, column: str +) -> np.ndarray: + """``values`` as the engine stores them: numpy's ``astype``, as core converts.""" + + try: + with np.errstate(invalid="ignore"): + return values.astype(stored_dtype) + except (TypeError, ValueError, OverflowError): + # numpy's own message can quote a value, so it is not chained. + raise TypeError( + f"The engine cannot store the {entity!r} ids in {column!r} " + f"({values.dtype}) as {stored_dtype}." + ) from None + + +def _exact( + values: np.ndarray, stored_dtype: np.dtype, *, entity: str, column: str +) -> bool: + """Whether every value converts to itself: a whole number inside the type. + + The bounds are compared in the values' own type, or in float64 for a + narrower float, so no comparison rounds. + """ + + if values.size == 0: + return True + info = np.iinfo(stored_dtype) + if values.dtype.kind in "iu": + if np.can_cast(values.dtype, stored_dtype, casting="safe"): + return True + held = np.iinfo(values.dtype) + low, high = max(info.min, held.min), min(info.max, held.max) + return bool(((values >= low) & (values <= high)).all()) + if values.dtype.kind == "f": + if values.dtype.itemsize < 8: + values = values.astype(np.float64) + with np.errstate(invalid="ignore"): + # Both bounds are powers of two, which a float holds exactly. + inside = (values >= info.min) & (values < info.max + 1) + whole = np.isfinite(values) & (np.floor(values) == values) + return bool((whole & inside).all()) + stored = _stored(values, stored_dtype, entity=entity, column=column) + return bool(np.all(stored.astype(object) == values.astype(object))) + + +def _ranks( + table: pd.DataFrame, entity: str, column: str +) -> tuple[np.ndarray, np.ndarray]: + """Each row's 1-based rank among the entity's ids, and the ids sorted.""" + + values = _complete_values(table, entity, column) + order = np.argsort(values, kind="stable") + ordered = values[order] + if bool((ordered[1:] == ordered[:-1]).any()): + raise ValueError(f"The {entity!r} ids in {column!r} repeat, so have no rank.") + ranks = np.empty(values.size, dtype=np.int64) + ranks[order] = np.arange(1, values.size + 1, dtype=np.int64) + return ranks, ordered + + +def _positions( + ordered: np.ndarray, + members: np.ndarray, + *, + group: str, + person_entity: str, + column: str, +) -> np.ndarray: + """Each member's 0-based position among the sorted group ids ``ordered``.""" + + left, right = _comparable(ordered, members) + positions = np.searchsorted(left, right) + found = positions < left.size + found[found] = left[positions[found]] == right[found] + if not found.all(): + raise ValueError( + f"The {person_entity!r} column {column!r} names a {group!r} id the " + f"{group!r} table does not hold." + ) + return positions.astype(np.int64) + + +def _comparable(left: np.ndarray, right: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """``left`` and ``right`` in one type that compares their values exactly. + + numpy compares int64 with uint64 through float64, which merges ids above + 2**53, so integers are brought to int64 or uint64 where both fit, and + anything else to Python objects. + """ + + if left.dtype == right.dtype: + return left, right + if left.dtype.kind in "iu" and right.dtype.kind in "iu": + if _fits_int64(left) and _fits_int64(right): + return left.astype(np.int64), right.astype(np.int64) + if _non_negative(left) and _non_negative(right): + return left.astype(np.uint64), right.astype(np.uint64) + return left.astype(object), right.astype(object) + + +def _fits_int64(values: np.ndarray) -> bool: + if values.dtype.kind == "i" or values.size == 0: + return True + return values.dtype.itemsize < 8 or int(values.max()) <= _INT64_MAX + + +def _non_negative(values: np.ndarray) -> bool: + return values.dtype.kind == "u" or values.size == 0 or int(values.min()) >= 0 diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py index bca07ab21f..2ae528c845 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/policyengine_uk.py @@ -8,6 +8,13 @@ geography (``UK_SPINE_ENGINE_BATCH_COLUMN``) is computed clone by clone, one simulation per clone, through the shared engine-block machinery (microcosm#931); every other frame takes one simulation, as it always has. + +policyengine-core stores the five id and membership columns as int32 +(``UK_ENGINE_IDENTIFIERS``). Each simulation is given the frame's ids where +that store keeps them apart, and 1 plus each id's rank among the simulation's +own ids for an entity whose stored ids would repeat +(``microcosm.frame.adapters.engine_identifiers``, microcosm#931). The frame is +never changed, and results return by position or by the frame's own ids. """ from __future__ import annotations @@ -16,15 +23,24 @@ import operator import sys import threading +import warnings from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from importlib import metadata from pathlib import Path -from types import MappingProxyType +from types import MappingProxyType, ModuleType from typing import Any, Protocol import numpy as np - +import pandas as pd + +from microcosm.frame.adapters.engine_identifiers import ( + EngineIdentifiers, + engine_identifier_tables, + identifier_columns, + refuse_inexact_identifier_requests, + require_stored_dtypes, +) from microcosm.frame.adapters.engine_population import ( SPINE_BATCHED_PASS, HouseholdBatch, @@ -50,6 +66,7 @@ "ProjectingRulesEngine", "SUPPLIED_ROLE_INPUTS", "UK_DEFAULT_SAVINGS_INCOME_PROJECTION", + "UK_ENGINE_IDENTIFIERS", "UK_ISA_INCOME_CONTRACT", "UK_ISA_INCOME_CONTRACT_ATTR", "UK_LEGACY_ISA_INCOME_CONTRACT", @@ -67,6 +84,16 @@ ] UK_SCHEMA = EntitySchema(group_entities=("benunit", "household")) +#: How policyengine-uk stores UK_SCHEMA's five id and membership columns. +#: policyengine-uk declares its id variables ``value_type = int``, which +#: policyengine-core stores as int32, converting a larger id with numpy's +#: ``astype`` without an error (microcosm#931). ``_require_identifier_dtypes`` +#: checks the declaration against the installed engine, and a test holds +#: national_frame's release range (``UK_ENGINE_IDENTIFIER_MAX``) to the same +#: width. national_frame's release rule keeps its own ranker; moving it onto +#: ``rank_identifier_tables``, and deriving that range from this declaration, +#: are owed. +UK_ENGINE_IDENTIFIERS = EngineIdentifiers(stored_dtype=np.dtype(np.int32)) #: How the producer's savings interest holds ISA interest, declared to #: policyengine-uk (its ``isa_income_contract``, from 2.162.2): non-ISA interest #: in ``savings_interest_income`` and ISA interest in @@ -403,6 +430,17 @@ class PolicyEngineUKEngine: the rows it holds, as a stage before the clones exist must; that no stage after them drops the column is the clone stage's integration check, not this adapter's. + + Each simulation is given the frame's ids where policyengine-core's + 32-bit store keeps them apart, and 1 plus each id's rank among that + simulation's ids for an entity whose stored ids would repeat, each + person's membership taking the new id of the group it names + (:meth:`_dataset_tables`). The frame is never changed, and results + return by position or by the frame's own ids. A request for an id or + membership variable the engine cannot return as the frame holds it is + refused, and each simulation is built with policyengine-uk's + ``EntityIdWarning`` filtered, since the adapter checks the stored ids + itself. """ country = "uk" @@ -420,6 +458,7 @@ def __init__(self, *, clones: int | None = None) -> None: ) clones = count self._system: Any = None + self._identifier_dtypes_checked = False self._clones = clones def variable_metadata(self, name: str) -> VariableMetadata: @@ -481,8 +520,25 @@ def materialize( declaration's guard judges what it ran under the spine's pass, which raises on anything only the pool defines, and the values come back as one simulation over the bundle returns them. + + Each simulation is given the frame's ids where its 32-bit store keeps + them apart, and 1 plus their rank among its own ids where it would + not. The frame is never changed, and values return by position or + by the frame's own ids. Because ranks are taken per simulation, + values the engine keys on ids, such as a draw hashed on them, can + differ between a batched call and one simulation over the bundle + whenever either side renumbers: a batch ranks its own rows 1 to n_b, + while one simulation over the bundle would rank 1 to N, and would + also renumber on ids that coincide across batches. Every other value + is as one simulation's. A request for an id or membership variable + whose entity holds an id the engine cannot store exactly is refused + before any engine is built. """ + variables = tuple(variables) + refuse_inexact_identifier_requests( + UK_SCHEMA, variables, UK_ENGINE_IDENTIFIERS, table=bundle.table + ) microsimulation_class = self._import_policyengine_uk().Microsimulation if dataset_period is not None and int(dataset_period) > int(period): raise ValueError( @@ -663,13 +719,16 @@ def _read_year_simulation( (:func:`uk_rebalancing_years_through`). A later year would change no value at ``period``: policyengine-uk builds each year only from earlier ones. + + Every simulation the adapter builds is constructed here, by + :meth:`_simulation`, inside the cut (microcosm#931). """ projected = uk_dataset_through( dataset, period, self._tax_benefit_system().parameters ) with uk_rebalancing_years_through(period): - return microsimulation_class(dataset=projected) + return self._simulation(microsimulation_class, projected) def _import_policyengine_uk(self) -> Any: try: @@ -681,6 +740,38 @@ def _import_policyengine_uk(self) -> Any: ) from exc return policyengine_uk + def _simulation(self, microsimulation_class: Any, dataset: Any) -> Any: + """One simulation over ``dataset``, built with policyengine-uk's + ``EntityIdWarning`` filtered, since the adapter itself checks the ids + each simulation stores (:meth:`_dataset_tables`). + + The warning class is read from the module + :meth:`_import_policyengine_uk` returns, at + ``utils.entity_ids.EntityIdWarning``, so the method adds no import of + its own, and the engine-free stubs that replace that method stand in + here too. A real module that defines no such class there is refused, + so a policyengine-uk that moved the class cannot bring the warning + back unnoticed. A stand-in that is not a module and carries no such + class constructs without a filter. + """ + + module = self._import_policyengine_uk() + warning: Any = module + for name in ("utils", "entity_ids", "EntityIdWarning"): + warning = getattr(warning, name, None) + if not (isinstance(warning, type) and issubclass(warning, Warning)): + if isinstance(module, ModuleType): + raise ImportError( + "The PolicyEngine-UK adapter builds its simulations with " + f"{module.__name__}.utils.entity_ids.EntityIdWarning " + "filtered, and the installed policyengine-uk defines no " + "such warning there; review the filter against the engine." + ) + return microsimulation_class(dataset=dataset) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=warning) + return microsimulation_class(dataset=dataset) + def _tax_benefit_system(self) -> Any: if self._system is None: self._system = self._import_policyengine_uk().CountryTaxBenefitSystem() @@ -697,8 +788,39 @@ def enum_domain(self, name: str) -> Any: return getattr(self._variable(name), "possible_values", None) - def _build_dataset(self, bundle: Frame, period: int | str) -> Any: - from policyengine_uk.data import UKSingleYearDataset + def _require_identifier_dtypes(self) -> None: + """Refuse an installed engine that stores an id or membership column + as a type other than ``UK_ENGINE_IDENTIFIERS`` declares. + + Read from this adapter's own ``CountryTaxBenefitSystem``. A pass is + remembered for the adapter; a refusal is raised again on every later + call. + """ + + if self._identifier_dtypes_checked: + return + require_stored_dtypes( + UK_ENGINE_IDENTIFIERS, + { + column: self._variable(column).dtype + for column in identifier_columns(UK_SCHEMA) + }, + ) + self._identifier_dtypes_checked = True + + def _dataset_tables(self, bundle: Frame) -> dict[str, pd.DataFrame]: + """The tables one simulation over ``bundle`` receives; no engine is + built. + + The frame's engine tables (``engine_tables``, with the household + weights), with each entity's ids as :func:`engine_identifier_tables` + gives them. An entity whose stored ids stay apart keeps the frame's, + so a bundle without a stored repeat gets exactly ``engine_tables``' + tables. An entity whose stored ids would repeat takes 1 plus each + id's rank among the bundle's ids, and each person's membership takes + its group's new id. A batch's bundle is ranked over its own rows. The + frame is not changed. + """ expected = (_PERSON_TABLE, *_GROUP_TABLES) if set(bundle.entities) != set(expected): @@ -707,6 +829,13 @@ def _build_dataset(self, bundle: Frame, period: int | str) -> Any: f"{list(expected)}; bundle has {list(bundle.entities)}." ) tables = engine_tables(bundle, weighted_entities=("household",)) + return engine_identifier_tables(UK_SCHEMA, tables, UK_ENGINE_IDENTIFIERS).tables + + def _build_dataset(self, bundle: Frame, period: int | str) -> Any: + from policyengine_uk.data import UKSingleYearDataset + + self._require_identifier_dtypes() + tables = self._dataset_tables(bundle) return UKSingleYearDataset( person=tables["person"].copy(), benunit=tables["benunit"].copy(), diff --git a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_adapter.py b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_adapter.py index a47b46b0d9..696f920756 100644 --- a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_adapter.py +++ b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_adapter.py @@ -7,8 +7,10 @@ def test_policyengine_uk_adapter_builds_a_real_engine_dataset() -> None: # Regression: the real UKSingleYearDataset constructor takes fiscal_year, # not time_period — a kwarg mismatch the protocol tests above cannot see - # because they never import the engine. Dataset construction is cheap - # (no simulation), so this runs wherever the uk extra is installed. + # because they never import the engine. Building the dataset runs no + # simulation, but it builds the adapter's CountryTaxBenefitSystem, whose + # id variables the adapter checks its int32 declaration against + # (microcosm#931). It runs wherever the uk extra is installed. import numpy as np import pandas as pd diff --git a/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_identifiers.py b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_identifiers.py new file mode 100644 index 0000000000..26dd181e7e --- /dev/null +++ b/packages/microcosm-frame/tests/engine/uk/test_policyengine_uk_identifiers.py @@ -0,0 +1,653 @@ +"""The ids each policyengine-uk simulation holds, through the adapter (microcosm#931). + +policyengine-core builds a simulation's entities and memberships from the +int64 id columns it is given, and stores those columns as the values of +policyengine-uk's five id variables, converted to int32 by numpy's +``astype``, which wraps an id outside the type without an error. Two ids that +wrap to one value therefore merge no entity, but every formula that reads the +stored id sees one key: policyengine-uk hashes its Warm Home Discount draw on +household_id, its Universal Credit deduction draws on benunit_id and its +birth-month spread on person_id. The adapter gives each simulation the +frame's ids where the store keeps them apart, and 1 plus each id's rank among +that simulation's own ids for an entity whose stored ids would repeat +(``microcosm.frame.adapters.engine_identifiers``). Held here through the +installed engine: + +- the adapter's int32 declaration is the engine's (test 11); +- ids past 2**31 that stay apart once stored, laid out as copy ids are, reach + the engine as they did before the rule: every value is a direct + Microsimulation's over the frame's engine tables, at 2024 and at 2025 as the + dataset's year and at 2025 projected from 2024, and the engine holds the + frame's rows in its order (test 12); +- households whose ids store as one value (7 and 7 + 2**32) take ranks, so + their draws differ and every value is the same call's over the frame + renumbered by hand: at the dataset's year (test 13), projected a year on + through policyengine-uk's uprating, whose council-tax and student-loan + joins read the raw memberships (test 13b), and clone by clone, each batch + ranking its own rows (test 14). All three fail where the adapter passes the + frame's ids through unchanged; +- policyengine-uk's ``EntityIdWarning`` is filtered at both of the adapter's + construction sites, and a direct Microsimulation over the same tables still + records it (test 15). + +Every household and id here is made up. Assertion messages name variables, +entities and periods only, never an id or a value. +""" + +from __future__ import annotations + +import warnings +from types import SimpleNamespace + +import numpy as np +import pytest + +from microcosm.frame import Frame +from microcosm.frame.adapters.engine_identifiers import ( + identifier_columns, + stored_identifier_repeats, +) +from microcosm.frame.adapters.engine_population import household_batches +from microcosm.frame.adapters.policyengine_uk import ( + UK_ENGINE_IDENTIFIERS, + UK_SCHEMA, + PolicyEngineUKEngine, + frame_isa_income_contract, + frame_savings_income_projection, +) +from microcosm.frame.adapters.policyengine_uk_population import ( + UK_SPINE_ENGINE_BATCH_COLUMN, +) +from microcosm.frame.materialize import engine_tables +from test_support.microcosm_frame.engine_identifier_frames import ( + COLLIDING_CLONE_HOUSEHOLD_IDS, + COLLIDING_HOUSEHOLD_IDS, + clone_layout_frame, + colliding_clones_frame, + colliding_household_frame, + copy_layout_frame, + relabelled, + wrapped, +) + +D, P = 2024, 2025 +INT32 = np.dtype(np.int32) +#: The id-keyed draws policyengine-uk hashes on stored ids: households', +#: benefit units' (two), and the birth-month spread on people's. +ID_DRAWS = ( + "warm_home_discount_random_draw", + "uc_deduction_random_draw", + "uc_deduction_type_random_draw", + "months_since_last_birthday", +) +#: Test 12's requests: the draws, values over the whole structure, each +#: household's member count and an echoed household input. +END_TO_END = ( + *ID_DRAWS, + "hbai_household_net_income", + "universal_credit", + "income_tax", + "household_count_people", + "council_tax", +) +#: Tests 13 and 13b's requests: test 12's, the two person inputs the uprating +#: joins rewrite, and the person and benefit-unit ids, which fit int32 and so +#: may be asked for. +COLLIDING = ( + *END_TO_END, + "council_tax_benefit_reported", + "student_loan_plan", + "person_id", + "benunit_id", +) +#: Test 14's requests, household-local and free of the pool's shares, as +#: test_policyengine_uk_batches.py splits LOCAL from SHARE_DEPENDENT_TAXES: a +#: standalone call over one batch gets no pool shares, so values that depend +#: on them are not compared. months_since_last_birthday is stored, as the +#: spine's batched pass requires. +BATCH_LOCAL = ( + *ID_DRAWS, + "income_tax", + "national_insurance", + "universal_credit", + "household_count_people", + "council_tax", +) + +#: The colliding frame's people, as colliding_household_frame lists them. +COLLIDING_PEOPLE = (34, 31, 33, 35, 36, 32) +#: Each of those people's inputs. People 31 and 36 hold no student loan, are +#: tertiary-educated and aged 31, an age at which policyengine-uk's +#: projection from 2024 to 2025 adds new holders (its extended 2012 intake), +#: and both draw a loan on their person ids, so the plan each takes follows +#: the region of its household: the first (Scotland) and the third (Wales). +#: The colliding households' adults report Council Tax Benefit, which the +#: projection carries by their household's council-tax factor. +COLLIDING_INPUTS = { + "age": {34: 38.0, 31: 31.0, 33: 45.0, 35: 12.0, 36: 31.0, 32: 6.0}, + "employment_income": { + 34: 31_000.0, + 31: 24_500.0, + 33: 18_000.0, + 35: 0.0, + 36: 12_000.0, + 32: 0.0, + }, + "highest_education": { + 34: "UPPER_SECONDARY", + 31: "TERTIARY", + 33: "TERTIARY", + 35: "PRIMARY", + 36: "TERTIARY", + 32: "PRIMARY", + }, + "student_loan_plan": dict.fromkeys(COLLIDING_PEOPLE, "NONE"), + "student_loan_repayments": dict.fromkeys(COLLIDING_PEOPLE, 0.0), + "council_tax_benefit_reported": { + 34: 185.5, + 31: 210.0, + 33: 0.0, + 35: 0.0, + 36: 96.25, + 32: 0.0, + }, +} +#: The colliding frame's households, in row order (7, 9 and 7 + 2**32): the +#: first and third in different nations, the second in Northern Ireland, +#: whose council tax policyengine-uk holds flat. +COLLIDING_HOUSEHOLDS = { + "region": ["SCOTLAND", "NORTHERN_IRELAND", "WALES"], + "council_tax": [1_210.0, 0.0, 1_450.25], + "rent": [0.0, 4_800.0, 7_800.0], + "tenure_type": ["OWNED_OUTRIGHT", "RENT_FROM_COUNCIL", "RENT_PRIVATELY"], +} + + +def _quiet(call, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return call(*args, **kwargs) + + +def _check(condition, what: str) -> None: + """Assert ``condition``; a failure names only ``what``, never the ids or + values the condition compared.""" + + holds = bool(condition) + assert holds, what + + +def _assert_same(left, right, what: str) -> None: + """Two returned arrays are equal bit for bit, and of one dtype; a failure + names only ``what``.""" + + left, right = np.asarray(left), np.asarray(right) + same = left.dtype == right.dtype and left.shape == right.shape + if same and left.dtype.kind == "O": + same = left.tolist() == right.tolist() + elif same: + same = left.tobytes() == right.tobytes() + assert same, f"{what} differs" + + +def _snapshot(frame: Frame) -> dict: + return {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + + +def _assert_unchanged(frame: Frame, before: dict) -> None: + for entity, table in before.items(): + unchanged = frame.table(entity).equals(table) and list( + frame.table(entity).dtypes + ) == list(table.dtypes) + assert unchanged, f"the frame's {entity} table changed" + + +def _with_columns(frame: Frame, entity: str, columns) -> Frame: + """``frame`` with ``columns`` added to its ``entity`` table, in row order.""" + + tables = {name: frame.table(name).copy() for name in frame.entities} + for column, values in columns.items(): + tables[entity][column] = list(values) + return Frame( + tables, + frame.schema, + {name: frame.weights_for(name) for name in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _household_inputs(count: int, **columns) -> dict: + """The household inputs policyengine-uk's projection of a dataset reads + (region, council tax, rent and tenure), one value each unless given.""" + + inputs = { + "region": ["LONDON"] * count, + "council_tax": [1_400.0] * count, + "rent": [0.0] * count, + "tenure_type": ["OWNED_OUTRIGHT"] * count, + } + inputs.update(columns) + return inputs + + +def _member_counts(frame: Frame) -> np.ndarray: + """Each household row's own number of people in the frame.""" + + members = frame.table("person")["person_household_id"].value_counts() + return frame.table("household")["household_id"].map(members).to_numpy() + + +def _dataset(frame: Frame, tables, fiscal_year: int): + """The dataset the adapter built before the rank rule: its engine tables' + copies, as they come.""" + + from policyengine_uk.data import UKSingleYearDataset + + return UKSingleYearDataset( + person=tables["person"].copy(), + benunit=tables["benunit"].copy(), + household=tables["household"].copy(), + fiscal_year=fiscal_year, + isa_income_contract=frame_isa_income_contract(frame), + savings_income_projection=frame_savings_income_projection(frame), + ) + + +def _direct_values(frame: Frame, names, period: int, dataset_period: int | None): + """``names`` from a Microsimulation built directly over the frame's + engine tables: the adapter's path before the rank rule.""" + + from policyengine_uk import Microsimulation + + tables = engine_tables(frame, weighted_entities=("household",)) + dataset = _dataset( + frame, tables, period if dataset_period is None else dataset_period + ) + simulation = _quiet(Microsimulation, dataset=dataset) + return { + name: np.asarray(_quiet(simulation.calculate, name, period=period)) + for name in names + } + + +def _entity_id_warned(call, *args, **kwargs) -> bool: + """Whether ``call`` raised policyengine-uk's ``EntityIdWarning``, under + ``simplefilter("always")``: a boolean, never the warning's text.""" + + from policyengine_uk.utils.entity_ids import EntityIdWarning + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + call(*args, **kwargs) + return any(issubclass(item.category, EntityIdWarning) for item in caught) + + +@pytest.fixture(scope="module") +def system(): + from policyengine_uk import CountryTaxBenefitSystem + + return CountryTaxBenefitSystem() + + +@pytest.fixture(scope="module") +def engine(): + return PolicyEngineUKEngine() + + +# --------------------------------------------------------------------------- +# Test 11: the declaration is the installed engine's +# --------------------------------------------------------------------------- + + +def test_the_id_declaration_is_the_installed_engines(system): + from policyengine_uk.utils.entity_ids import ENTITY_ID_VARIABLES + + columns = identifier_columns(UK_SCHEMA) + assert len(ENTITY_ID_VARIABLES) == len(set(ENTITY_ID_VARIABLES)) == len(columns) + assert set(ENTITY_ID_VARIABLES) == set(columns) + for column in columns: + variable = system.variables[column] + assert variable.value_type is int, column + assert np.dtype(variable.dtype) == UK_ENGINE_IDENTIFIERS.stored_dtype, column + assert UK_ENGINE_IDENTIFIERS.stored_dtype == INT32 + adapter = PolicyEngineUKEngine() + adapter._system = system + adapter._require_identifier_dtypes() + + +# --------------------------------------------------------------------------- +# Test 12: ids that stay apart once stored reach the engine as before +# --------------------------------------------------------------------------- + + +def _copy_layout_frame() -> Frame: + """Test 12's frame: every entity's ids laid out as copy ids are, six + households of different sizes in different regions, adults in work.""" + + frame = copy_layout_frame( + household_columns=_household_inputs( + 6, + region=[ + "LONDON", + "SCOTLAND", + "WALES", + "NORTH_WEST", + "SOUTH_EAST", + "NORTHERN_IRELAND", + ], + council_tax=[1_210.0, 1_307.5, 1_405.0, 1_502.5, 1_600.0, 1_697.5], + rent=[0.0, 6_600.0, 0.0, 8_400.0, 0.0, 5_100.0], + tenure_type=[ + "OWNED_OUTRIGHT", + "RENT_PRIVATELY", + "OWNED_WITH_MORTGAGE", + "RENT_PRIVATELY", + "OWNED_OUTRIGHT", + "RENT_FROM_COUNCIL", + ], + ) + ) + age = frame.table("person")["age"].to_numpy() + pay = 7_000.0 + 2_750.0 * np.arange(len(age)) + return _with_columns( + frame, "person", {"employment_income": np.where(age >= 18, pay, 0.0)} + ) + + +@pytest.mark.parametrize( + ("period", "dataset_period"), + [(D, None), (P, None), (P, D)], + ids=["2024", "2025", "2025_projected_from_2024"], +) +def test_ids_that_stay_apart_once_stored_give_the_direct_simulations_values( + engine, period, dataset_period +): + # The end-to-end half of the proof that a frame without a stored repeat + # reaches the engine unchanged: ids past 2**31 that stay apart once + # stored are handed to the engine as they were before the rank rule, so + # every value, id-keyed draws included, is a direct Microsimulation's over + # the frame's engine tables. + frame = _copy_layout_frame() + tables = engine_tables(frame, weighted_entities=("household",)) + # The layout's own claims: every entity's ids pass 2**31 and stay apart + # once stored, and the households' stored ids are out of int64 order. + assert stored_identifier_repeats(UK_SCHEMA, tables, UK_ENGINE_IDENTIFIERS) == () + for entity in UK_SCHEMA.entities: + ids = tables[entity][UK_SCHEMA.entity_id_column(entity)] + _check(int(ids.max()) > 2**31, f"the {entity} ids fit int32") + stored = [wrapped(value, INT32) for value in tables["household"]["household_id"]] + _check(stored != sorted(stored), "the household ids store in int64 order") + before = _snapshot(frame) + + values = _quiet( + engine.materialize, frame, END_TO_END, period, dataset_period=dataset_period + ) + + direct = _direct_values(frame, END_TO_END, period, dataset_period) + for name in END_TO_END: + _assert_same(values[name], direct[name], f"{name} at {period}") + # Frame truth, which the comparison cannot give, since an engine that + # built its groups from stored ids would reorder both sides alike: each + # household row's own member count (all different), and at the dataset's + # year an echoed household input with distinct values, in row order. + counts = _member_counts(frame) + _check(len(set(counts.tolist())) == len(counts), "two households are one size") + _assert_same( + values["household_count_people"].astype(np.int64), + counts.astype(np.int64), + "household_count_people against the frame's members", + ) + if dataset_period is None: + council_tax = frame.table("household")["council_tax"].to_numpy() + _check( + len(set(council_tax.tolist())) == len(council_tax), + "two households pay one council tax", + ) + _assert_same( + values["council_tax"].astype(np.float64), + council_tax, + "council_tax against the frame's column", + ) + _check(np.any(values["income_tax"] != 0), f"no income tax at {period}") + _assert_unchanged(frame, before) + + +# --------------------------------------------------------------------------- +# Tests 13 and 13b: households whose ids store as one value +# --------------------------------------------------------------------------- + + +def _colliding_frame(household_ids=COLLIDING_HOUSEHOLD_IDS) -> Frame: + """Tests 13 and 13b's frame: households 7, 9 and 7 + 2**32, or + ``household_ids`` for the same frame renumbered by hand.""" + + return colliding_household_frame( + household_ids, + person_columns={ + column: [by_person[person] for person in COLLIDING_PEOPLE] + for column, by_person in COLLIDING_INPUTS.items() + }, + household_columns=COLLIDING_HOUSEHOLDS, + ) + + +def _colliding_runs(engine, period: int, dataset_period: int | None): + """One call over the colliding frame, and the same call over it + renumbered by hand to households 1, 2 and 3.""" + + frame = _colliding_frame() + # The layout's own claims: the people are listed as the inputs assume, + # and the first and third households store as one int32 value. + _check( + tuple(frame.table("person")["person_id"].tolist()) == COLLIDING_PEOPLE, + "the people are not listed as the inputs assume", + ) + first, _second, third = COLLIDING_HOUSEHOLD_IDS + _check( + wrapped(first, INT32) == wrapped(third, INT32), + "the first and third households store apart", + ) + before = _snapshot(frame) + values = _quiet( + engine.materialize, frame, COLLIDING, period, dataset_period=dataset_period + ) + hand = _quiet( + engine.materialize, + _colliding_frame((1, 2, 3)), + COLLIDING, + period, + dataset_period=dataset_period, + ) + return SimpleNamespace(frame=frame, before=before, values=values, hand=hand) + + +def _assert_ranked(runs, period: int) -> None: + """What tests 13 and 13b share.""" + + values, hand, frame = runs.values, runs.hand, runs.frame + draws = values["warm_home_discount_random_draw"] + # The colliding households' draws differ: without the rank rule both hash + # the one stored id and are equal. + _check( + draws[0] != draws[2], f"the colliding households' draws are equal at {period}" + ) + # Ranks keep order and change nothing else: every value is the hand + # renumbering's, bit for bit. + for name in COLLIDING: + _assert_same(values[name], hand[name], f"{name} at {period}") + _assert_same( + values["household_count_people"].astype(np.int64), + _member_counts(frame).astype(np.int64), + "household_count_people against the frame's members", + ) + _assert_unchanged(frame, runs.before) + + +def test_households_whose_ids_store_together_take_ranks(engine): + # Test 13, at the dataset's year. + runs = _colliding_runs(engine, P, None) + _assert_ranked(runs, P) + council_tax = runs.frame.table("household")["council_tax"].to_numpy() + _assert_same( + runs.values["council_tax"].astype(np.float64), + council_tax, + "council_tax against the frame's column", + ) + + +def test_households_whose_ids_store_together_take_ranks_when_projected(engine): + # Test 13b: as test 13, projected from 2024, as uc_take_up and + # pension_credit_take_up call. policyengine-uk's uprating then maps each + # household's council-tax factor onto its people by person_household_id, + # and merges region onto people by it for new student-loan holders: joins + # on the raw membership column, which a membership map that missed its + # group's new id would misattach. + runs = _colliding_runs(engine, P, D) + _assert_ranked(runs, P) + # The joins are exercised: the colliding households' council-tax factors + # differ, and their new borrowers take different plans. + row = {person: index for index, person in enumerate(COLLIDING_PEOPLE)} + reported = runs.values["council_tax_benefit_reported"].astype(np.float64) + given = COLLIDING_INPUTS["council_tax_benefit_reported"] + factors = [reported[row[person]] / given[person] for person in (31, 34)] + _check(factors[0] != factors[1], "the colliding households share a factor") + plans = runs.values["student_loan_plan"] + _check(plans[row[31]] != plans[row[36]], "the new borrowers take one plan") + _check( + "NONE" not in (plans[row[31]], plans[row[36]]), "a new borrower takes no plan" + ) + + +# --------------------------------------------------------------------------- +# Test 14: clone by clone, each batch ranks its own rows +# --------------------------------------------------------------------------- + + +def _clones_frame() -> Frame: + """Test 14's frame: two clones, each holding two households whose ids + store as one int32 value, adults in work.""" + + return colliding_clones_frame( + person_columns={ + "employment_income": [26_000.0, 0.0, 38_500.0, 14_250.0, 0.0, 47_000.0] + }, + household_columns=_household_inputs( + 4, + region=["LONDON", "LONDON", "WALES", "WALES"], + council_tax=[1_380.0, 1_392.5, 1_455.0, 1_467.5], + ), + ) + + +def _scatter(frame: Frame, entity: str, parts) -> np.ndarray: + """Per-batch values on ``frame``'s ``entity`` rows, each placed by the + batch frame's own id.""" + + column = UK_SCHEMA.entity_id_column(entity) + placed = {} + for batch_ids, values in parts: + for identifier, value in zip(batch_ids, values, strict=True): + _check(identifier not in placed, f"two batches return one {entity} row") + placed[identifier] = value + ids = [int(value) for value in frame.table(entity)[column]] + _check(set(placed) == set(ids), f"the batches miss a {entity} row") + return np.asarray( + [placed[identifier] for identifier in ids], dtype=parts[0][1].dtype + ) + + +def test_each_batch_ranks_its_own_rows(engine): + # Test 14: within each clone's batch the two households' ids store as one + # value (7, and 1000000007 for clone 1), while people and benefit units + # pass 2**31 and stay apart. Each batch ranks its own households, so its + # values are those of a standalone call over the batch renumbered by hand + # to households 1 and 2. + frame = _clones_frame() + before = _snapshot(frame) + _check(engine.simulation_batches(frame) == 2, "the frame runs in other batches") + + values = _quiet(engine.materialize, frame, BATCH_LOCAL, P) + + batches = household_batches(frame, UK_SPINE_ENGINE_BATCH_COLUMN) + _check( + [batch.key for batch in batches] == list(COLLIDING_CLONE_HOUSEHOLD_IDS), + "the batches follow other clones", + ) + draws = values["warm_home_discount_random_draw"] + entities = {name: engine.variable_metadata(name).entity for name in BATCH_LOCAL} + parts = {name: [] for name in BATCH_LOCAL} + for batch, households in zip( + batches, COLLIDING_CLONE_HOUSEHOLD_IDS.values(), strict=True + ): + part = frame.select(batch.persons) + _check( + part.table("household")["household_id"].tolist() == list(households), + "a batch holds other households", + ) + _check( + len({wrapped(value, INT32) for value in households}) == 1, + "a batch's households store apart", + ) + # The batch's two draws differ: without the rank rule both hash the + # one stored id and are equal. + own = draws[batch.households] + _check(own[0] != own[1], f"a batch's two draws are equal at {P}") + alone = _quiet( + engine.materialize, relabelled(part, "household", (1, 2)), BATCH_LOCAL, P + ) + for name in BATCH_LOCAL: + entity = entities[name] + ids = part.table(entity)[UK_SCHEMA.entity_id_column(entity)] + parts[name].append(([int(value) for value in ids], alone[name])) + for name in BATCH_LOCAL: + expected = _scatter(frame, entities[name], parts[name]) + _assert_same(values[name], expected, f"{name} at {P}") + _check(np.any(values["income_tax"] != 0), f"no income tax at {P}") + _assert_unchanged(frame, before) + + +# --------------------------------------------------------------------------- +# Test 15: the warning is filtered at both construction sites +# --------------------------------------------------------------------------- + + +def test_the_id_warning_is_filtered_where_one_simulation_is_built(engine): + # Test 15, unbatched: the shifted clone layout, whose ids pass 2**31 and + # stay apart. + from policyengine_uk import Microsimulation + + frame = clone_layout_frame() + frame = _with_columns(frame, "household", _household_inputs(frame.n("household"))) + tables = engine_tables(frame, weighted_entities=("household",)) + # policyengine-uk still warns on ids outside int32 ... + _check( + _entity_id_warned(Microsimulation, dataset=_dataset(frame, tables, P)), + "a direct Microsimulation no longer warns", + ) + # ... and the adapter's simulation does not pass it on. + _check( + not _entity_id_warned(engine.materialize, frame, ["income_tax"], P), + "the adapter's simulation passed the warning on", + ) + + +def test_the_id_warning_is_filtered_where_batches_are_built(engine): + # Test 15, batched: test 14's frame, whose people pass 2**31 in every + # batch. This is the site the spine's calls over more than one clone use. + from policyengine_uk import Microsimulation + + frame = _clones_frame() + batches = household_batches(frame, UK_SPINE_ENGINE_BATCH_COLUMN) + part = frame.select(batches[0].persons) + tables = engine_tables(part, weighted_entities=("household",)) + _check( + _entity_id_warned(Microsimulation, dataset=_dataset(part, tables, P)), + "a direct Microsimulation over one batch no longer warns", + ) + _check(engine.simulation_batches(frame) == 2, "the frame runs in other batches") + _check( + not _entity_id_warned(engine.materialize, frame, ["income_tax"], P), + "the adapter's batched simulations passed the warning on", + ) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_engine_identifiers.py b/packages/microcosm-frame/tests/engine_free/shared/test_engine_identifiers.py new file mode 100644 index 0000000000..859229177c --- /dev/null +++ b/packages/microcosm-frame/tests/engine_free/shared/test_engine_identifiers.py @@ -0,0 +1,547 @@ +"""The ids each engine simulation holds for a frame (microcosm#931). + +``microcosm.frame.adapters.engine_identifiers`` hands a simulation the tables it +was given where no entity's ids repeat once the engine stores them, and gives +an entity whose stored ids would repeat 1 plus each id's rank, each membership +taking its group's new id. Held here on neutral schemas: ids that stay apart +pass through as the same objects, a repeat renumbers only its entity, a seeded +sweep over ids of every NumPy and nullable integer dtype keeps order and +memberships, an id request the engine cannot answer is refused without reading +a table it need not, and every refusal names entities, columns and types, never +an id or a count. Every id here is made up. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from microcosm.frame import EntitySchema, Frame, LinkSpec, WeightKind, Weights +from microcosm.frame.adapters import engine_identifiers +from microcosm.frame.adapters.engine_identifiers import ( + EngineIdentifiers, + EngineIdentifierTables, + engine_identifier_tables, + identifier_columns, + inexact_identifier_entities, + rank_identifier_tables, + refuse_inexact_identifier_requests, + require_stored_dtypes, + stored_identifier_repeats, +) +from test_support.microcosm_frame.engine_identifier_frames import ( + COLLIDING_HOUSEHOLD_IDS, + FAMILY_SCHEMA, + NEUTRAL_SCHEMAS, + SWEEP_STORED_DTYPES, + WRAP, + colliding_household_tables, + layout_tables, + sweep_frame, + sweep_tables, + wrapped, +) + +INT32 = EngineIdentifiers(np.int32) +#: Sweep cases per schema and stored type. +SWEEP_CASES = 100 + + +def _schema_id(schema) -> str: + return "-".join(schema.group_entities) + + +def _copies(tables): + return {entity: table.copy(deep=True) for entity, table in tables.items()} + + +def _assert_unchanged(tables, before) -> None: + assert list(tables) == list(before) + for entity, table in tables.items(): + pd.testing.assert_frame_equal(table, before[entity], check_exact=True) + + +def _id_values(schema, tables) -> dict[str, list[int]]: + """Every id and membership column's values, as Python ints in row order.""" + + person = tables[schema.person_entity] + values = {} + for entity in schema.entities: + column = schema.entity_id_column(entity) + values[column] = [int(value) for value in tables[entity][column]] + for group in schema.group_entities: + column = schema.membership_column(group) + values[column] = [int(value) for value in person[column]] + return values + + +def _order(values: list[int]) -> list[int]: + return sorted(range(len(values)), key=values.__getitem__) + + +def _assert_count_free(message: str, *numbers: int | float) -> None: + """A refusal quotes none of the frame's multi-digit ids, wrapped values or + row counts (single digits are never checked: a type name can hold one). + A failure names none of them either.""" + + quoted = any(str(number) in message for number in numbers) + assert not quoted, "the refusal quotes an id, a wrapped value or a row count" + + +# --------------------------------------------------------------------------- +# Test 1: ids that stay apart once stored pass through. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("layout", ["inside", "copies"]) +@pytest.mark.parametrize("schema", NEUTRAL_SCHEMAS, ids=_schema_id) +def test_ids_that_stay_apart_once_stored_pass_through(schema, layout): + tables = layout_tables(schema, layout) + before = _copies(tables) + stored = { + column: [wrapped(value, np.int32) for value in values] + for column, values in _id_values(schema, tables).items() + } + # The layout's own claims: every entity's ids stay apart once stored; the + # copies pass 2**31 for every entity, and some wrap negative. + for entity in schema.entities: + column = schema.entity_id_column(entity) + assert len(set(stored[column])) == len(stored[column]) + if layout == "copies": + assert max(_id_values(schema, tables)[column]) > 2**31 + assert min(stored[column]) < 0 + assert stored_identifier_repeats(schema, tables, INT32) == () + + result = engine_identifier_tables(schema, tables, INT32) + + assert isinstance(result, EngineIdentifierTables) + assert result.renumbered == () + assert list(result.tables) == list(tables) + for entity, table in tables.items(): + assert result.tables[entity] is table + _assert_unchanged(tables, before) + # Passing through does not mean the engine holds them exactly. + assert inexact_identifier_entities(schema, tables, INT32) == ( + () if layout == "inside" else schema.entities + ) + + +# --------------------------------------------------------------------------- +# Test 2: a stored repeat renumbers its entity by rank. +# --------------------------------------------------------------------------- + + +def test_a_stored_repeat_renumbers_that_entity_by_rank(): + tables = colliding_household_tables() + before = _copies(tables) + first, second, third = COLLIDING_HOUSEHOLD_IDS + assert wrapped(first, np.int32) == wrapped(third, np.int32) + assert stored_identifier_repeats(FAMILY_SCHEMA, tables, INT32) == ("household",) + + result = engine_identifier_tables(FAMILY_SCHEMA, tables, INT32) + + assert result.renumbered == ("household",) + household = result.tables["household"] + assert household["household_id"].tolist() == [1, 2, 3] + assert household["household_id"].dtype == np.dtype(np.int64) + assert household["rent"].tolist() == tables["household"]["rent"].tolist() + # Each person's membership is their household's engine id, 1 plus its + # rank, never the 0-based rank. + engine_id = {first: 1, second: 2, third: 3} + person = result.tables["person"] + frame_memberships = tables["person"]["person_household_id"].tolist() + assert person["person_household_id"].tolist() == [ + engine_id[value] for value in frame_memberships + ] + assert person["person_household_id"].tolist() == [3, 1, 2, 1, 3, 2] + assert person["person_household_id"].dtype == np.dtype(np.int64) + # People keep their order, ids and other memberships. + for column in ("person_id", "person_family_id", "age"): + pd.testing.assert_series_equal(person[column], tables["person"][column]) + assert result.tables["family"] is tables["family"] + assert household is not tables["household"] + assert person is not tables["person"] + assert list(result.tables) == list(tables) + assert stored_identifier_repeats(FAMILY_SCHEMA, result.tables, INT32) == () + _assert_unchanged(tables, before) + + +# --------------------------------------------------------------------------- +# Test 3: a seeded sweep over ids of every NumPy and nullable integer dtype. +# --------------------------------------------------------------------------- + + +def _assert_engine_ids(frame, given, returned, *, renumbered, stored_dtype) -> None: + """``returned`` holds ``frame``'s ids with exactly ``renumbered`` ranked. + + An engine id is the value in the returned table, not its stored form: an + entity that passes through outside the type stores in another order. + """ + + schema = frame.schema + engine = _id_values(schema, returned) + assert list(returned) == list(given) + for entity in schema.entities: + column = schema.entity_id_column(entity) + frame_ids, engine_ids = frame.ids[entity], engine[column] + # Sorting rows by engine id is sorting them by frame id. + assert _order(engine_ids) == _order(frame_ids) + if entity in renumbered: + assert sorted(engine_ids) == list(range(1, len(frame_ids) + 1)) + assert returned[entity][column].dtype == np.dtype(np.int64) + else: + assert engine_ids == frame_ids + assert returned[entity][column].dtype == given[entity][column].dtype + if stored_dtype is not None: + stored = [wrapped(value, stored_dtype) for value in engine_ids] + assert len(set(stored)) == len(stored), entity + value_column = f"{entity}_value" + pd.testing.assert_series_equal( + returned[entity][value_column], given[entity][value_column] + ) + person = schema.person_entity + for group in schema.group_entities: + column = schema.membership_column(group) + # Each engine membership is the engine id of the group the frame's + # membership names; a 0-based map fails this. + engine_of = dict( + zip(frame.ids[group], engine[schema.id_column(group)], strict=True) + ) + assert engine[column] == [ + engine_of[value] for value in frame.memberships[group] + ] + expected = ( + np.dtype(np.int64) if group in renumbered else given[person][column].dtype + ) + assert returned[person][column].dtype == expected + assert (returned[group] is given[group]) == (group not in renumbered) + assert (returned[person] is given[person]) == (not renumbered) + + +@pytest.mark.parametrize("stored_dtype", SWEEP_STORED_DTYPES, ids=str) +@pytest.mark.parametrize("schema", NEUTRAL_SCHEMAS, ids=_schema_id) +def test_a_seeded_sweep_ranks_exactly_the_entities_whose_stored_ids_repeat( + schema, stored_dtype +): + identifiers = EngineIdentifiers(stored_dtype) + rng = np.random.default_rng([931, len(schema.entities), stored_dtype.itemsize]) + decisions = set() + for _case in range(SWEEP_CASES): + frame = sweep_frame(rng, schema) + tables = sweep_tables(frame, rng) + before = _copies(tables) + expected = tuple( + entity + for entity in schema.entities + if len({wrapped(value, stored_dtype) for value in frame.ids[entity]}) + < len(frame.ids[entity]) + ) + inexact = tuple( + entity + for entity in schema.entities + if any(wrapped(value, stored_dtype) != value for value in frame.ids[entity]) + ) + assert stored_identifier_repeats(schema, tables, identifiers) == expected + assert inexact_identifier_entities(schema, tables, identifiers) == inexact + assert set(expected) <= set(inexact) + + result = engine_identifier_tables(schema, tables, identifiers) + + # An entity renumbers exactly when its stored ids repeat, and after, + # no entity's do. + assert result.renumbered == expected + _assert_engine_ids( + frame, tables, result.tables, renumbered=expected, stored_dtype=stored_dtype + ) + # The same ids in other drawn dtypes: the same decision and ranks. + retyped = sweep_tables(frame, rng) + again = engine_identifier_tables(schema, retyped, identifiers) + assert again.renumbered == expected + assert _id_values(schema, again.tables) == _id_values(schema, result.tables) + # The rank step ranks exactly the entities it is given. + chosen = tuple(entity for entity in schema.entities if rng.random() < 0.5) + ranked = rank_identifier_tables(schema, tables, chosen) + _assert_engine_ids(frame, tables, ranked, renumbered=chosen, stored_dtype=None) + assert _id_values(schema, rank_identifier_tables(schema, retyped, chosen)) == ( + _id_values(schema, ranked) + ) + _assert_unchanged(tables, before) + decisions.add(bool(expected)) + # The sweep reaches both branches. + assert decisions == {False, True} + + +# --------------------------------------------------------------------------- +# Test 4: inexact entities, and id requests the engine cannot answer. +# --------------------------------------------------------------------------- + + +def _inexact_tables(): + """People fit int32; household 9 + 2**32 lies outside it; family 1.5 is + not a whole number. Neither entity's stored ids repeat.""" + + outside = 9 + WRAP + return { + "person": pd.DataFrame( + { + "person_id": np.asarray([21, 22, 23], dtype=np.int64), + "person_family_id": np.asarray([1.5, 3.0, 3.0]), + "person_household_id": np.asarray([7, outside, 7], dtype=np.int64), + } + ), + "family": pd.DataFrame({"family_id": np.asarray([1.5, 3.0])}), + "household": pd.DataFrame( + {"household_id": np.asarray([7, outside], dtype=np.int64)} + ), + } + + +def _reader(tables, reads): + def table(entity): + reads.append(entity) + return tables[entity] + + return table + + +def test_inexact_entities_and_the_id_requests_the_engine_cannot_answer(): + assert dict(identifier_columns(FAMILY_SCHEMA)) == { + "person_id": "person", + "family_id": "family", + "person_family_id": "family", + "household_id": "household", + "person_household_id": "household", + } + assert list(identifier_columns(FAMILY_SCHEMA)) == [ + "person_id", + "family_id", + "person_family_id", + "household_id", + "person_household_id", + ] + # A group whose id column is another group's membership column. + clashing = EntitySchema(group_entities=("household", "person_household")) + with pytest.raises(ValueError, match="'person_household_id' twice"): + identifier_columns(clashing) + tables = _inexact_tables() + assert stored_identifier_repeats(FAMILY_SCHEMA, tables, INT32) == () + assert inexact_identifier_entities(FAMILY_SCHEMA, tables, INT32) == ( + "family", + "household", + ) + # An entity whose stored ids repeat is inexact too. + assert inexact_identifier_entities( + FAMILY_SCHEMA, colliding_household_tables(), INT32 + ) == ("household",) + + refused = { + "household_id": "household", + "person_household_id": "household", + "family_id": "family", + "person_family_id": "family", + } + for name, entity in refused.items(): + reads = [] + with pytest.raises(ValueError, match="cannot return") as excinfo: + refuse_inexact_identifier_requests( + FAMILY_SCHEMA, + iter([name, "age"]), + INT32, + table=_reader(tables, reads), + ) + message = str(excinfo.value) + assert repr(name) in message and repr(entity) in message + assert "int32" in message + _assert_count_free(message, 9 + WRAP, 1.5, 21, 22, 23) + assert reads == [entity] + + # The persons fit, and a variable that is no id needs no table at all. + reads = [] + refuse_inexact_identifier_requests( + FAMILY_SCHEMA, ["person_id", "age"], INT32, table=_reader(tables, reads) + ) + assert reads == ["person"] + + def refuse_to_read(entity): + raise AssertionError("no table is read for a request with no id") + + refuse_inexact_identifier_requests( + FAMILY_SCHEMA, ["age", "income"], INT32, table=refuse_to_read + ) + refuse_inexact_identifier_requests(FAMILY_SCHEMA, [], INT32, table=refuse_to_read) + + # Each entity is read once, and only the inexact entities' columns are + # named. + reads = [] + with pytest.raises(ValueError) as excinfo: + refuse_inexact_identifier_requests( + FAMILY_SCHEMA, + ["person_id", "household_id", "person_household_id", "household_id"], + INT32, + table=_reader(tables, reads), + ) + assert reads == ["person", "household"] + message = str(excinfo.value) + assert "['household_id', 'person_household_id']" in message + assert "'person_id'" not in message + with pytest.raises(TypeError, match="not one name"): + refuse_inexact_identifier_requests( + FAMILY_SCHEMA, "household_id", INT32, table=refuse_to_read + ) + + +# --------------------------------------------------------------------------- +# Test 5: refusals name entities, columns and types, never an id or a count. +# --------------------------------------------------------------------------- + + +def test_an_entity_with_more_rows_than_the_type_numbers_is_refused(): + # 200 people (more than int8 numbers from 1), two of whose ids store as one + # value in int8: 1000 and 1256 both store as -24. + person_ids = [1000 + index for index in range(199)] + [1256] + assert wrapped(1000, np.int8) == wrapped(1256, np.int8) == -24 + tables = { + "person": pd.DataFrame( + { + "person_id": np.asarray(person_ids, dtype=np.int64), + "person_family_id": np.ones(200, dtype=np.int64), + "person_household_id": np.asarray([1, 2] * 100, dtype=np.int64), + } + ), + "family": pd.DataFrame({"family_id": np.asarray([1], dtype=np.int64)}), + "household": pd.DataFrame({"household_id": np.asarray([1, 2], dtype=np.int64)}), + } + identifiers = EngineIdentifiers(np.int8) + assert stored_identifier_repeats(FAMILY_SCHEMA, tables, identifiers) == ("person",) + with pytest.raises(ValueError, match="more rows than int8") as excinfo: + engine_identifier_tables(FAMILY_SCHEMA, tables, identifiers) + message = str(excinfo.value) + assert "'person'" in message + _assert_count_free(message, 200, 199, 1000, 1198, 1256, -24, 232, 127, 128) + + +@pytest.mark.parametrize( + "stored_dtype", + [np.float64, "float32", np.bool_, pd.Int32Dtype(), object, "no such type"], + ids=str, +) +def test_a_declaration_that_is_no_integer_type_is_refused(stored_dtype): + with pytest.raises(TypeError, match="must be a NumPy integer type"): + EngineIdentifiers(stored_dtype) + + +def test_a_declaration_is_normalised_and_held_to_the_engine(): + assert EngineIdentifiers("int32").stored_dtype == np.dtype(np.int32) + assert EngineIdentifiers(np.int32) == EngineIdentifiers(np.dtype("int32")) + assert EngineIdentifiers(np.uint16).stored_dtype == np.dtype(np.uint16) + require_stored_dtypes( + INT32, + {"person_id": np.int32, "family_id": np.dtype("int32"), "household_id": "i4"}, + ) + require_stored_dtypes(INT32, {}) + with pytest.raises(ValueError, match="declared int32") as excinfo: + require_stored_dtypes( + INT32, + { + "person_id": np.int32, + "household_id": np.int64, + "person_household_id": pd.Int32Dtype(), + }, + ) + message = str(excinfo.value) + assert "'household_id' as int64" in message + assert "'person_household_id' as Int32Dtype()" in message + assert "'person_id'" not in message + with pytest.raises(TypeError, match="EngineIdentifiers declaration"): + require_stored_dtypes(np.dtype(np.int32), {"person_id": np.int32}) + + +def test_engine_ids_that_still_repeat_after_ranking_raise(monkeypatch): + # Stands for a later edit that broke the rank step: it now returns the + # tables as given. + monkeypatch.setattr( + engine_identifiers, + "rank_identifier_tables", + lambda schema, tables, entities: dict(tables), + ) + with pytest.raises(RuntimeError, match="still repeat") as excinfo: + engine_identifier_tables(FAMILY_SCHEMA, colliding_household_tables(), INT32) + message = str(excinfo.value) + assert "'household'" in message and "int32" in message + _assert_count_free(message, 7 + WRAP) + + +def test_the_rank_step_refuses_what_it_cannot_rank(): + tables = colliding_household_tables() + with pytest.raises(ValueError, match="undeclared entities") as excinfo: + rank_identifier_tables(FAMILY_SCHEMA, tables, ["household", "tax_unit"]) + assert "['tax_unit']" in str(excinfo.value) + with pytest.raises(TypeError, match="not one"): + rank_identifier_tables(FAMILY_SCHEMA, tables, "household") + # A membership that names no household (a Frame rules it out). + stray = 11 + WRAP + person = tables["person"].copy() + person.loc[0, "person_household_id"] = stray + with pytest.raises(ValueError, match="does not hold") as excinfo: + rank_identifier_tables( + FAMILY_SCHEMA, {**tables, "person": person}, ["household"] + ) + _assert_count_free(str(excinfo.value), stray, 7 + WRAP) + # Ids the tables themselves repeat have no rank (a Frame rules it out too). + household = pd.DataFrame( + {"household_id": np.asarray([9, 9 + WRAP, 9 + WRAP], dtype=np.int64)} + ) + with pytest.raises(ValueError, match="repeat, so have no rank") as excinfo: + rank_identifier_tables( + FAMILY_SCHEMA, {**tables, "household": household}, ["household"] + ) + _assert_count_free(str(excinfo.value), 9 + WRAP) + # A declared link's table that names an entity to renumber: the step maps + # only the person table's memberships, so it refuses rather than hand + # back a link table holding the old ids. + linked = EntitySchema( + group_entities=("family", "household"), + links=(LinkSpec("visits", "person", "household"),), + ) + visits = pd.DataFrame( + { + "person_id": np.asarray([11, 13], dtype=np.int64), + "household_id": np.asarray([7, 7 + WRAP], dtype=np.int64), + } + ) + with_link = {**tables, "visits": visits} + with pytest.raises(ValueError, match="link tables") as excinfo: + rank_identifier_tables(linked, with_link, ["household"]) + message = str(excinfo.value) + assert "['household']" in message and "['visits']" in message + _assert_count_free(message, 7 + WRAP, 11, 13) + with pytest.raises(ValueError, match="link tables"): + engine_identifier_tables(linked, with_link, INT32) + # A link that names no entity to renumber, or whose table is not among + # the tables, is left as it is. + assert rank_identifier_tables(linked, with_link, ["family"])["visits"] is visits + assert engine_identifier_tables(linked, tables, INT32).renumbered == ("household",) + # Nothing to rank: the same objects back. + unranked = rank_identifier_tables(FAMILY_SCHEMA, tables, []) + assert all(unranked[entity] is table for entity, table in tables.items()) + + +def test_the_functions_take_tables_never_a_frame(): + frame = Frame( + colliding_household_tables(), + FAMILY_SCHEMA, + {"household": Weights(values=np.ones(3), kind=WeightKind.DESIGN)}, + ) + with pytest.raises(TypeError, match="map each entity to its table"): + stored_identifier_repeats(FAMILY_SCHEMA, frame, INT32) + with pytest.raises(ValueError, match="no 'household' table"): + engine_identifier_tables( + FAMILY_SCHEMA, + { + entity: table + for entity, table in colliding_household_tables().items() + if entity != "household" + }, + INT32, + ) diff --git a/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py b/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py index 368b80eb0d..3cd2f2860a 100644 --- a/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py +++ b/packages/microcosm-frame/tests/engine_free/uk/test_policyengine_uk_adapter.py @@ -1,13 +1,31 @@ """Tests split from packages/microcosm-frame/tests/test_policyengine_uk_adapter.py.""" # ruff: noqa: F403, F405 +import sys +import types +import warnings + from microcosm.frame import EntitySchema, Frame, WeightKind, Weights from microcosm.frame.adapters import policyengine_uk as adapter_module +from microcosm.frame.adapters.engine_identifiers import ( + identifier_columns, + inexact_identifier_entities, + stored_identifier_repeats, +) from microcosm.frame.adapters.engine_population import SPINE_BATCHED_PASS +from microcosm.frame.adapters.policyengine_uk import UK_ENGINE_IDENTIFIERS from microcosm.frame.adapters.policyengine_uk_population import ( UK_ENGINE_POPULATION, UK_SPINE_ENGINE_BATCH_COLUMN, ) +from microcosm.frame.materialize import engine_tables +from test_support.microcosm_frame.engine_identifier_frames import ( + COLLIDING_CLONE_HOUSEHOLD_IDS, + clone_layout_frame, + colliding_clones_frame, + relabelled, + wrapped, +) from test_support.microcosm_frame.policyengine_uk_adapter import * @@ -738,3 +756,424 @@ def test_the_parameters_are_the_engines_own(): engine = PolicyEngineUKEngine() engine._system = SimpleNamespace(parameters=lambda period: ("tree at", period)) assert engine.parameters(2025) == ("tree at", 2025) + + +# --------------------------------------------------------------------------- +# The ids each simulation holds (microcosm#931) +# --------------------------------------------------------------------------- + + +def _identifier_system(dtype=np.int32, **dtypes): + """A stand-in tax-benefit system whose five id variables store as + ``dtype``, or as ``dtypes[column]``.""" + + return SimpleNamespace( + variables={ + column: SimpleNamespace(dtype=dtypes.get(column, dtype)) + for column in identifier_columns(UK_SCHEMA) + } + ) + + +def _ids(frame) -> list[int]: + """Every id of every entity of ``frame``, as Python ints.""" + + return [ + int(value) + for entity in UK_SCHEMA.entities + for value in frame.table(entity)[UK_SCHEMA.entity_id_column(entity)] + ] + + +def _assert_count_free(message: str, numbers) -> None: + """A refusal quotes none of the frame's multi-digit ids or their stored + values (single digits are never checked: a type name can hold one). A + failure names none of them either.""" + + quoted = any(str(number) in message for number in numbers if abs(number) >= 10) + assert not quoted, "the refusal quotes an id or a stored value" + + +def _snapshot(frame) -> dict[str, pd.DataFrame]: + return {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + + +def _assert_unchanged(frame, before) -> None: + for entity, table in before.items(): + pd.testing.assert_frame_equal(frame.table(entity), table, check_exact=True) + + +def test_the_engine_declaration_is_int32_and_held_to_the_engine_once(monkeypatch): + # policyengine-uk declares its five id variables ``value_type = int``, + # which policyengine-core stores as int32. + assert UK_ENGINE_IDENTIFIERS.stored_dtype == np.dtype("int32") + assert dict(identifier_columns(UK_SCHEMA)) == { + "person_id": "person", + "benunit_id": "benunit", + "person_benunit_id": "benunit", + "household_id": "household", + "person_household_id": "household", + } + # The adapter holds the declaration to its own engine once: a second + # check would build an engine, which _no_engine refuses. + engine = PolicyEngineUKEngine() + _no_engine(monkeypatch, engine) + engine._system = _identifier_system() + engine._require_identifier_dtypes() + engine._system = None + engine._require_identifier_dtypes() + # An engine that stores an id column otherwise is refused, every time. + other = PolicyEngineUKEngine() + _no_engine(monkeypatch, other) + other._system = _identifier_system(person_household_id=np.int64) + for _ in range(2): + with pytest.raises(ValueError) as excinfo: + other._require_identifier_dtypes() + assert "'person_household_id' as int64, not as the declared int32" in str( + excinfo.value + ) + # The dataset builder holds the declaration before it builds anything. + built = [] + data = types.ModuleType("policyengine_uk.data") + data.UKSingleYearDataset = lambda **kwargs: built.append(kwargs) + monkeypatch.setitem(sys.modules, "policyengine_uk.data", data) + with pytest.raises(ValueError, match="declared int32"): + other._build_dataset(clone_layout_frame(), 2025) + assert built == [] + + +def test_ids_that_stay_apart_once_stored_reach_the_engine_as_its_tables(monkeypatch): + # The structural half of the proof that a frame without a stored repeat + # reaches the engine unchanged: a frame whose ids pass 2**31 but stay + # apart once stored, as K = 1's SPI copies do, reaches the engine as + # exactly the tables it did before. + frame = clone_layout_frame() + before = _snapshot(frame) + tables = engine_tables(frame, weighted_entities=("household",)) + # The layout's own claims. + for entity in UK_SCHEMA.entities: + ids = [int(value) for value in tables[entity][f"{entity}_id"]] + assert min(ids) > 2**31 + assert len({wrapped(value, np.int32) for value in ids}) == len(ids) + assert stored_identifier_repeats(UK_SCHEMA, tables, UK_ENGINE_IDENTIFIERS) == () + assert ( + inexact_identifier_entities(UK_SCHEMA, tables, UK_ENGINE_IDENTIFIERS) + == UK_SCHEMA.entities + ) + engine = PolicyEngineUKEngine() + _no_engine(monkeypatch, engine) + made = [] + + def recording(bundle, **kwargs): + made.append(engine_tables(bundle, **kwargs)) + return made[-1] + + monkeypatch.setattr(adapter_module, "engine_tables", recording) + + given = engine._dataset_tables(frame) + + # The very objects engine_tables made, not equal copies. + (returned,) = made + assert list(given) == list(returned) + assert all(given[entity] is returned[entity] for entity in returned) + assert list(given) == list(tables) + for entity, table in tables.items(): + pd.testing.assert_frame_equal(given[entity], table, check_exact=True) + # And the dataset is built from those tables, after the declaration is + # checked, with no engine built. + built = [] + data = types.ModuleType("policyengine_uk.data") + data.UKSingleYearDataset = lambda **kwargs: built.append(kwargs) or "dataset" + monkeypatch.setitem(sys.modules, "policyengine_uk.data", data) + engine._system = _identifier_system() + assert engine._build_dataset(frame, 2025) == "dataset" + (kwargs,) = built + assert kwargs["fiscal_year"] == 2025 + for entity, table in tables.items(): + pd.testing.assert_frame_equal(kwargs[entity], table, check_exact=True) + _assert_unchanged(frame, before) + + +def test_each_batch_ranks_its_own_rows_where_its_household_ids_store_together( + monkeypatch, +): + # Each clone holds two households whose ids store as one int32 value; + # each batch numbers its own households 1 to n in id order. + frame = colliding_clones_frame() + before = _snapshot(frame) + engine = PolicyEngineUKEngine() + _no_engine(monkeypatch, engine) + batches = engine._clone_batches(frame) + assert [batch.key for batch in batches] == list(COLLIDING_CLONE_HOUSEHOLD_IDS) + for batch, households in zip( + batches, COLLIDING_CLONE_HOUSEHOLD_IDS.values(), strict=True + ): + part = frame.select(batch.persons) + tables = engine_tables(part, weighted_entities=("household",)) + # The layout's own claims: the batch's households store together, + # and nothing else repeats once stored. + assert part.table("household")["household_id"].tolist() == list(households) + assert len({wrapped(value, np.int32) for value in households}) == 1 + assert stored_identifier_repeats(UK_SCHEMA, tables, UK_ENGINE_IDENTIFIERS) == ( + "household", + ) + + given = engine._dataset_tables(part) + + # Exactly the batch renumbered by hand: households 1 and 2, each + # membership its household's new id. + engine_id = {household: rank + 1 for rank, household in enumerate(households)} + hand = engine_tables( + relabelled(part, "household", list(engine_id.values())), + weighted_entities=("household",), + ) + assert list(given) == list(hand) + for entity in UK_SCHEMA.entities: + pd.testing.assert_frame_equal(given[entity], hand[entity], check_exact=True) + assert given["household"]["household_id"].tolist() == [1, 2] + assert given["person"]["person_household_id"].tolist() == [ + engine_id[int(value)] for value in tables["person"]["person_household_id"] + ] + # Persons and benefit units pass through; nothing but the two + # household id columns moves. + pd.testing.assert_frame_equal( + given["benunit"], tables["benunit"], check_exact=True + ) + for entity, column in ( + ("person", "person_household_id"), + ("household", "household_id"), + ): + pd.testing.assert_frame_equal( + given[entity].drop(columns=column), + tables[entity].drop(columns=column), + check_exact=True, + ) + _assert_unchanged(frame, before) + + +def test_each_batch_dataset_is_ranked_over_its_own_rows(monkeypatch): + # As above, through materialize's batched site and _build_dataset: the + # batches are handed the frame itself, and each batch's dataset numbers + # that batch's households 1 and 2, people and benefit units passing + # through. + frame = colliding_clones_frame() + before = _snapshot(frame) + built = [] + data = types.ModuleType("policyengine_uk.data") + data.UKSingleYearDataset = lambda **kwargs: built.append(kwargs) or "dataset" + monkeypatch.setitem(sys.modules, "policyengine_uk.data", data) + engine = PolicyEngineUKEngine() + engine._system = _identifier_system() + monkeypatch.setattr( + engine, + "_import_policyengine_uk", + lambda: SimpleNamespace(Microsimulation=lambda dataset: SimpleNamespace()), + ) + # The engine's year window needs policyengine-uk itself; the stand-in + # builds the dataset as it is, through the adapter's own construction. + monkeypatch.setattr( + engine, + "_read_year_simulation", + lambda microsimulation_class, dataset, period: engine._simulation( + microsimulation_class, dataset + ), + ) + handed = [] + + def batched(bundle, variables, period, **kwargs): + handed.append(bundle) + for batch in kwargs["batches"]: + kwargs["build"](bundle.select(batch.persons)) + return {} + + monkeypatch.setattr(adapter_module, "materialize_in_batches", batched) + engine.materialize(frame, ["age"], 2025) + assert len(handed) == 1 and handed[0] is frame + batches = engine._clone_batches(frame) + assert len(built) == len(batches) == len(COLLIDING_CLONE_HOUSEHOLD_IDS) + for batch, kwargs in zip(batches, built, strict=True): + part = engine_tables( + frame.select(batch.persons), weighted_entities=("household",) + ) + engine_id = { + int(household): rank + 1 + for rank, household in enumerate(part["household"]["household_id"]) + } + assert kwargs["fiscal_year"] == 2025 + assert kwargs["household"]["household_id"].tolist() == [1, 2] + assert kwargs["person"]["person_household_id"].tolist() == [ + engine_id[int(value)] for value in part["person"]["person_household_id"] + ] + for entity in ("person", "benunit"): + column = f"{entity}_id" + assert kwargs[entity][column].tolist() == part[entity][column].tolist() + _assert_unchanged(frame, before) + + +@pytest.mark.parametrize("build", [clone_layout_frame, colliding_clones_frame]) +@pytest.mark.parametrize( + "variables", + [["household_id"], ["age", "person_household_id"], ["benunit_id"]], +) +def test_an_id_request_the_engine_cannot_answer_is_refused_before_any_engine( + monkeypatch, build, variables +): + # The engine would hold wrapped ids, or ranks for the colliding clones' + # households, for these entities, not the frame's. + frame = build() + engine = PolicyEngineUKEngine() + _no_engine(monkeypatch, engine) + with pytest.raises(ValueError, match="cannot return") as excinfo: + engine.materialize(frame, variables, 2025) + message = str(excinfo.value) + for name in variables: + if name != "age": + assert repr(name) in message + ids = _ids(frame) + _assert_count_free(message, ids + [wrapped(value, np.int32) for value in ids]) + + +def test_an_id_request_the_engine_answers_reaches_it_with_every_name(monkeypatch): + # The person ids fit int32, though the households' and benefit units' do + # not: a person id request reaches the engine, and so does every name of + # a one-shot iterator, on either path. + engine, built = _one_simulation_engine(monkeypatch) + frame = clone_layout_frame(shifted=("benunit", "household")) + values = engine.materialize(frame, ["person_id"], 2025) + assert len(built) == 1 and list(values) == ["person_id"] + values = engine.materialize(frame, iter(["age", "person_id"]), 2025) + assert len(built) == 2 and list(values) == ["age", "person_id"] + calls = [] + monkeypatch.setattr( + adapter_module, + "materialize_in_batches", + lambda bundle, variables, period, **kwargs: calls.append(list(variables)), + ) + engine.materialize(_clone_frame((0, 1, 2)), iter(["age", "person_id"]), 2025) + assert calls == [["age", "person_id"]] + + +def _construction_engine(monkeypatch, module): + """An adapter whose engine module is ``module`` and whose datasets are + stubs; batched calls hand the first batch to its build closure.""" + + engine = PolicyEngineUKEngine() + monkeypatch.setattr(engine, "_import_policyengine_uk", lambda: module) + # The engine's year window needs policyengine-uk itself; the stand-in + # builds the dataset as it is, through the adapter's own construction. + monkeypatch.setattr( + engine, + "_read_year_simulation", + lambda microsimulation_class, dataset, period: engine._simulation( + microsimulation_class, dataset + ), + ) + monkeypatch.setattr( + engine, "_build_dataset", lambda bundle, period: list(range(bundle.n("person"))) + ) + monkeypatch.setattr( + engine, "variable_metadata", lambda name: SimpleNamespace(entity="person") + ) + + def batched(bundle, variables, period, **kwargs): + kwargs["build"](bundle.select(kwargs["batches"][0].persons)) + return {} + + monkeypatch.setattr(adapter_module, "materialize_in_batches", batched) + return engine + + +def _simulation_class(seen, emit=()): + """A stand-in Microsimulation recording the warning filters it was + constructed under, and emitting a made-up warning of each ``emit`` + category while it is constructed.""" + + class Simulation: + tax_benefit_system = SimpleNamespace(variables={}) + input_variables = () + + def __init__(self, dataset): + seen.append(list(warnings.filters)) + for category in emit: + warnings.warn("made-up construction warning", category, stacklevel=2) + self._rows = len(dataset) + + def calculate(self, name, period=None): + return np.zeros(self._rows) + + return Simulation + + +def _module_without_the_warning(shape): + module = types.ModuleType("policyengine_uk") + if shape in {"no_entity_ids", "not_a_warning"}: + module.utils = types.ModuleType("policyengine_uk.utils") + if shape == "not_a_warning": + module.utils.entity_ids = types.ModuleType("policyengine_uk.utils.entity_ids") + module.utils.entity_ids.EntityIdWarning = object + return module + + +@pytest.mark.parametrize("shape", ["no_utils", "no_entity_ids", "not_a_warning"]) +@pytest.mark.parametrize("clones", [None, (0, 1, 2)]) +def test_an_engine_module_without_the_id_warning_class_builds_nothing( + monkeypatch, shape, clones +): + # The filter fails closed at both construction sites, before any + # simulation is constructed. + seen = [] + module = _module_without_the_warning(shape) + module.Microsimulation = _simulation_class(seen) + engine = _construction_engine(monkeypatch, module) + frame = _clone_frame(column=False) if clones is None else _clone_frame(clones) + with pytest.raises(ImportError) as excinfo: + engine.materialize(frame, ["age"], 2025) + assert "policyengine_uk.utils.entity_ids.EntityIdWarning" in str(excinfo.value) + assert seen == [] + + +@pytest.mark.parametrize("clones", [None, (0, 1, 2)]) +def test_a_stand_in_engine_constructs_without_a_filter(monkeypatch, clones): + # A stand-in that is not a module and carries no warning class, as the + # stubs above are, constructs under the filters it was called with. + seen = [] + stand_in = SimpleNamespace(Microsimulation=_simulation_class(seen)) + engine = _construction_engine(monkeypatch, stand_in) + frame = _clone_frame(column=False) if clones is None else _clone_frame(clones) + outer = list(warnings.filters) + engine.materialize(frame, ["age"], 2025) + assert seen == [outer] + + +@pytest.mark.parametrize("clones", [None, (0, 1, 2)]) +def test_the_id_warning_is_filtered_only_while_a_simulation_is_built( + monkeypatch, clones +): + # The warning class is read from the engine module the adapter imported; + # other warnings pass, and the filter ends with the construction. + class EntityIdWarning(UserWarning): + pass + + seen = [] + module = types.ModuleType("policyengine_uk") + module.utils = SimpleNamespace( + entity_ids=SimpleNamespace(EntityIdWarning=EntityIdWarning) + ) + module.Microsimulation = _simulation_class( + seen, emit=(EntityIdWarning, RuntimeWarning) + ) + engine = _construction_engine(monkeypatch, module) + frame = _clone_frame(column=False) if clones is None else _clone_frame(clones) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + engine.materialize(frame, ["age"], 2025) + warnings.warn("made-up later warning", EntityIdWarning, stacklevel=1) + assert len(seen) == 1 + assert [ + (item.category, str(item.message)) + for item in caught + if str(item.message).startswith("made-up") + ] == [ + (RuntimeWarning, "made-up construction warning"), + (EntityIdWarning, "made-up later warning"), + ] diff --git a/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_parity.py b/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_parity.py index e7c3da10af..43a7c5174c 100644 --- a/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_parity.py +++ b/packages/microcosm-graph/tests/engine/uk/test_acceptance_h_parity.py @@ -7,7 +7,7 @@ _TEST_PATHS = paths_for("microcosm-graph") -def test_h2_uk_spine_parity(tmp_path: Path) -> None: +def test_h2_uk_spine_parity(tmp_path: Path, monkeypatch) -> None: """The UK spine as a graph reproduces ``uk_frame_content_identity``. The spine's stages read the UK engine's parameters, so this runs in the @@ -21,15 +21,37 @@ def test_h2_uk_spine_parity(tmp_path: Path) -> None: topological order is derived, so the hand-maintained ``_STAGE_NAMES`` tuple in ``tools/build_uk_frs_spine.py`` — the 30 names intersected with a 34-stage packaged manifest, kept in step by hand — can be deleted. + + Every simulation the UK adapter builds on either side, all at one clone, + must pass the frame's ids through (microcosm#931): no entity's ids may + repeat once policyengine-core stores them as int32, so each simulation + receives the tables it received before the adapter's rank rule, and no + value here moves with it. The run records that as booleans only. """ _require(UK_SPINE_PARITY, "the UK migration lane (charter H2, María reviews)") from microcosm.build.uk_runtime.content_identity import uk_frame_content_identity from microcosm.build.uk_runtime.graph import uk_registry, uk_spine_graph from microcosm.build.uk_runtime.graph_kernels import fixture_stage_plan_inputs + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from microcosm.graph import ContentStore, compile_graph, graph_from_json, run_graph from tools.graph_uk_spine_fixture import legacy_oracle_frame + # Whether the adapter's rank rule ran, and whether any simulation on + # either side renumbered an entity: two booleans, never an id or a count. + ids_seen = {"ranked": False, "renumbered": False} + rank_rule = uk_engine_adapter.engine_identifier_tables + + def recorded_rank_rule(schema, tables, identifiers): + given = rank_rule(schema, tables, identifiers) + ids_seen["ranked"] = True + ids_seen["renumbered"] = ids_seen["renumbered"] or bool(given.renumbered) + return given + + monkeypatch.setattr( + uk_engine_adapter, "engine_identifier_tables", recorded_rank_rule + ) + # The identity is a byte-exact fingerprint of every cell, and the FRS root # transform's weights differ at the last bit between machines, so both # sides derive the root from the same raw tables here, in this process: @@ -58,6 +80,9 @@ def test_h2_uk_spine_parity(tmp_path: Path) -> None: resume="forbid", decisions=(), ) + # Both sides have run: read the record before the failure branch below + # runs the oracle again. + ranked, renumbered = ids_seen["ranked"], ids_seen["renumbered"] final_version = compiled.versions[compiled.order[-1]] final = manifest.population(final_version) actual = uk_frame_content_identity(final) @@ -74,6 +99,14 @@ def test_h2_uk_spine_parity(tmp_path: Path) -> None: print(f"H2 graph {actual} != legacy {expected}\n{stability}\n{differences}") pytest.fail(f"graph != legacy oracle; {stability}\n{differences}") + # microcosm#931: every simulation must have passed its ids through, so + # that the rank rule moved no value here. + assert ranked, "no simulation reached the adapter's rank rule" + assert not renumbered, ( + "a simulation at one clone renumbered an entity whose ids repeat once " + "stored as int32" + ) + # Charter F12: stage order is derived from declared inputs, so the # hand-maintained tuple in the UK driver is gone. driver = (_TEST_PATHS.repository / "tools" / "build_uk_frs_spine.py").read_text() diff --git a/packages/microcosm-graph/tests/engine_after_merge/uk/test_h2_clone_first.py b/packages/microcosm-graph/tests/engine_after_merge/uk/test_h2_clone_first.py index efc6221694..d8d4cfb61d 100644 --- a/packages/microcosm-graph/tests/engine_after_merge/uk/test_h2_clone_first.py +++ b/packages/microcosm-graph/tests/engine_after_merge/uk/test_h2_clone_first.py @@ -45,10 +45,15 @@ fits at K = 4, and none at one clone. It cannot catch within-batch int32 collisions: ids past 2**31 reach the engine -at K = 1 already, and these households are too few (a licensed build's -per-batch int32 count is that check). Invented data only; assertion messages -give K, stages, columns, counts and booleans, never a weight, a value or an -id. +at K = 1 already, and these households are too few. The adapter's rank rule +handles them (microcosm.frame.adapters.engine_identifiers): an entity whose +ids would repeat once stored takes 1 plus each id's rank within its batch, +and microcosm-frame's test_engine_identifiers.py and +test_policyengine_uk_identifiers.py pin that on invented frames, so a +licensed build's per-batch int32 count is no longer the only check. Invented +data only; +assertion messages give K, stages, columns, counts and booleans, never a +weight, a value or an id. """ from __future__ import annotations diff --git a/test_support/microcosm_frame/engine_identifier_frames.py b/test_support/microcosm_frame/engine_identifier_frames.py new file mode 100644 index 0000000000..1249529e30 --- /dev/null +++ b/test_support/microcosm_frame/engine_identifier_frames.py @@ -0,0 +1,604 @@ +"""Invented frames and id layouts for the engine-identifier tests (microcosm#931). + +``microcosm.frame.adapters.engine_identifiers`` gives each engine simulation +the frame's ids where the engine's stored type keeps them apart, and 1 plus +their rank where it would not. Its shared tests, the UK adapter's engine-free +tests and the UK engine tier share these layouts: + +- :data:`COPY_IDS`: source ids and copies carrying the ordinals 10**11, 10**12 + and 10**13, laid out as a frame's copy ids are. The copies pass 2**31, all + six stay apart once stored as int32, and 3 + 10**12 wraps negative, so they + convert out of int64 order. :data:`COPY_GROUP_IDS` and + :data:`COPY_PERSON_IDS` lay out other entities' ids the same way; +- :data:`COLLIDING_HOUSEHOLD_IDS`: 7, 9 and 7 + 2**32, the first and last of + which store as 7; +- :data:`COLLIDING_CLONE_HOUSEHOLD_IDS`: two clones, each holding two + households whose ids store together (h and h + 2**32, plus 10**9 in clone + 1); +- two neutral schemas, and a seeded sweep of small tables whose id columns + take any NumPy or nullable integer dtype that holds their values + (``concept_id_dtypes.ID_DTYPES``). + +:func:`wrapped` is the tests' own reading of numpy's conversion: an integer's +low bits, two's complement. The UK builders import the UK adapter's names only +when called, so a shared test that imports this module loads no UK module. +Every id and value here is made up. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import NamedTuple + +import numpy as np +import pandas as pd + +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from test_support.microcosm_frame.concept_id_dtypes import ( + ID_DTYPES, + INT64_MAX, + INT64_MIN, + UINT64_MAX, + holds, + typed, +) + +#: The span of a 32-bit store: two ids this far apart store as one value. +WRAP = 2**32 +#: The offset the invented clone layouts add to every id of clone 1. +CLONE_OFFSET = 10**9 +#: The shift that carries an invented layout's ids past 2**31. +SHIFT = 10**11 + +#: Ids laid out as copy ids are (source ids 3 and 5, and copies carrying the +#: ordinals 10**11, 10**12 and 10**13), ascending. +COPY_IDS = (3, 5, 3 + 10**11, 5 + 10**11, 3 + 10**12, 3 + 10**13) +#: The same layout from the source ids 4 and 6, for a second group entity. +COPY_GROUP_IDS = (4, 6, 4 + 10**11, 6 + 10**11, 4 + 10**12, 4 + 10**13) +#: The same layout from the source ids 1 and 2, for a third group entity. +COPY_THIRD_GROUP_IDS = (1, 2, 1 + 10**11, 2 + 10**11, 1 + 10**12, 1 + 10**13) +#: Twenty-one person ids in the same layout: the odd source ids 11 to 23, each +#: also carrying the ordinals 10**11 and 10**12. +COPY_PERSON_IDS = tuple( + source + ordinal for ordinal in (0, 10**11, 10**12) for source in range(11, 24, 2) +) +#: Ids of the same shapes inside int32, which store as themselves. +INSIDE_IDS = (2, 3, 5, 7, 11, 13) +INSIDE_GROUP_IDS = (1, 4, 6, 8, 9, 10) +INSIDE_THIRD_GROUP_IDS = (12, 14, 15, 16, 18, 20) +INSIDE_PERSON_IDS = tuple(range(101, 122)) +#: Members of each of the six households in the layouts above, in id order: +#: all different, so a household matched to another's row would show. +LAYOUT_MEMBERS = (3, 1, 4, 2, 6, 5) + +#: Three households, the first and last of which store as 7 in int32. +COLLIDING_HOUSEHOLD_IDS = (7, 9, 7 + WRAP) +#: Two clones of two households each; within each clone both ids store as one +#: int32 value (7 for clone 0, 1000000007 for clone 1). +COLLIDING_CLONE_HOUSEHOLD_IDS: Mapping[int, tuple[int, int]] = { + 0: (7, 7 + WRAP), + 1: (7 + CLONE_OFFSET, 7 + WRAP + CLONE_OFFSET), +} + +#: A person entity and two groups, and one with a third group. +FAMILY_SCHEMA = EntitySchema(group_entities=("family", "household")) +THREE_GROUP_SCHEMA = EntitySchema(group_entities=("family", "household", "dwelling")) +NEUTRAL_SCHEMAS = (FAMILY_SCHEMA, THREE_GROUP_SCHEMA) + +#: The stored types the sweep declares: narrow, so repeats are common. +SWEEP_STORED_DTYPES = (np.dtype(np.int8), np.dtype(np.int16)) + +_LAYOUTS: Mapping[str, Mapping[str, tuple[int, ...]]] = { + "inside": { + "person": INSIDE_PERSON_IDS, + "household": INSIDE_IDS, + "family": INSIDE_GROUP_IDS, + "dwelling": INSIDE_THIRD_GROUP_IDS, + }, + "copies": { + "person": COPY_PERSON_IDS, + "household": COPY_IDS, + "family": COPY_GROUP_IDS, + "dwelling": COPY_THIRD_GROUP_IDS, + }, +} +#: Ages by place in a household of the layouts: two adults, then children. +_AGES = (40.0, 38.0, 12.0, 9.0, 6.0, 3.0) +_ROW_SEED = 931 + + +def wrapped(value: int, dtype: np.dtype) -> int: + """``value`` as numpy's ``astype`` stores it in the integer ``dtype``. + + Its low bits, read as two's complement for a signed type: the tests' + reading of the conversion, written apart from numpy's. + """ + + dtype = np.dtype(dtype) + bits = 8 * dtype.itemsize + low = int(value) % 2**bits + if dtype.kind == "i" and low >= 2 ** (bits - 1): + low -= 2**bits + return low + + +def _people(person_ids: Sequence[int], members: Sequence[int]): + """(person id, household place, place in the household) per person row. + + The ids are taken in order, household by household, then the rows are + listed in a fixed scrambled order, so they are not in id order. + """ + + if sum(members) != len(person_ids): + raise ValueError("The layout's members do not match its person ids.") + rows = [] + ids = iter(person_ids) + for household, size in enumerate(members): + for place in range(size): + rows.append((next(ids), household, place)) + order = np.random.default_rng(_ROW_SEED).permutation(len(rows)) + return [rows[index] for index in order] + + +# --------------------------------------------------------------------------- +# Neutral tables +# --------------------------------------------------------------------------- + + +def layout_tables(schema: EntitySchema, layout: str) -> dict[str, pd.DataFrame]: + """Six households' tables, their ids ``"inside"`` int32 or laid out as + ``"copies"`` are. + + Six households of :data:`LAYOUT_MEMBERS` people; family ``i`` is household + ``i``'s, and a third group spreads the people over its six ids. Every id is + int64, groups ascending, people out of id order. + """ + + ids = _LAYOUTS[layout] + rows = _people(ids["person"], LAYOUT_MEMBERS) + person: dict[str, object] = { + "person_id": np.asarray([row[0] for row in rows], dtype=np.int64) + } + places = { + "family": [row[1] for row in rows], + "household": [row[1] for row in rows], + "dwelling": [(5 * position) % 6 for position, _row in enumerate(rows)], + } + for group in schema.group_entities: + group_ids = ids[group] + person[schema.membership_column(group)] = np.asarray( + [group_ids[place] for place in places[group]], dtype=np.int64 + ) + person["age"] = np.asarray([_AGES[row[2]] for row in rows]) + tables = {"person": pd.DataFrame(person)} + for group in schema.group_entities: + group_ids = np.asarray(ids[group], dtype=np.int64) + tables[group] = pd.DataFrame( + {schema.id_column(group): group_ids, f"{group}_size": np.arange(6) + 1.5} + ) + return tables + + +def colliding_household_tables() -> dict[str, pd.DataFrame]: + """Tables of households 7, 9 and 7 + 2**32, people out of id order. + + The first and last households store as 7. Families and people fit int32. + """ + + first, second, third = COLLIDING_HOUSEHOLD_IDS + person = pd.DataFrame( + { + "person_id": np.asarray([13, 11, 15, 12, 14, 16], dtype=np.int64), + "person_family_id": np.asarray([4, 1, 3, 2, 4, 3], dtype=np.int64), + "person_household_id": np.asarray( + [third, first, second, first, third, second], dtype=np.int64 + ), + "age": [40.0, 38.0, 12.0, 9.0, 6.0, 3.0], + } + ) + family = pd.DataFrame( + {"family_id": np.asarray([1, 2, 3, 4], dtype=np.int64), "rooms": 3.0} + ) + household = pd.DataFrame( + { + "household_id": np.asarray(COLLIDING_HOUSEHOLD_IDS, dtype=np.int64), + "rent": [700.0, 650.0, 900.0], + } + ) + return {"person": person, "family": family, "household": household} + + +# --------------------------------------------------------------------------- +# The seeded sweep +# --------------------------------------------------------------------------- + + +class SweepFrame(NamedTuple): + """One sweep case's invented ids, as Python ints in row order. + + ``ids[entity]`` lists an entity's ids in its table's row order (groups + ascending, as a Frame keeps them; people in any order), and + ``memberships[group]`` each person row's id of ``group``. Every group has a + member. Each entity's ids lie wholly in int64's range or wholly in + uint64's, so some integer dtype holds them. + """ + + schema: EntitySchema + ids: dict[str, list[int]] + memberships: dict[str, list[int]] + + +#: Values within one of a power of two where int8, int16 or int32 wraps, and +#: the 64-bit extremes. +_SWEEP_EDGES = ( + 127, + 128, + 255, + 256, + 32767, + 32768, + 65535, + 65536, + 2**31 - 1, + 2**31, + 2**32 - 1, + 2**32, + 2**63 - 1, + 2**63, + 2**64 - 1, +) +#: Spans whose multiples wrap to one value in int8 (all three) or int16 (the +#: last two). +_SWEEP_SPANS = (256, 65536, WRAP) +_SWEEP_ORDINALS = (CLONE_OFFSET, 10**11, 10**12, 10**13) + + +def _sweep_ids(rng: np.random.Generator, count: int) -> list[int]: + """``count`` distinct ids, many a span apart from another so they wrap.""" + + unsigned = bool(rng.random() < 0.2) + low, high = (0, UINT64_MAX) if unsigned else (INT64_MIN, INT64_MAX) + values: list[int] = [] + seen: set[int] = set() + while len(values) < count: + branch = int(rng.integers(0, 5 if unsigned else 4)) + if branch == 1 and values: + base = values[int(rng.integers(0, len(values)))] + span = _SWEEP_SPANS[int(rng.integers(0, len(_SWEEP_SPANS)))] + value = base + int(rng.integers(-2, 3)) * span + elif branch == 2: + edge = _SWEEP_EDGES[int(rng.integers(0, len(_SWEEP_EDGES)))] + value = edge if unsigned or rng.random() < 0.5 else -edge + elif branch == 3: + ordinal = _SWEEP_ORDINALS[int(rng.integers(0, len(_SWEEP_ORDINALS)))] + value = int(rng.integers(0, 301)) + ordinal + elif branch == 4: + value = UINT64_MAX - int(rng.integers(0, 2**62)) + else: + value = int(rng.integers(0 if unsigned else -300, 301)) + if low <= value <= high and value not in seen: + seen.add(value) + values.append(value) + return values + + +def sweep_frame( + rng: np.random.Generator, schema: EntitySchema, *, max_people: int = 40 +) -> SweepFrame: + """A sweep case: up to ``max_people`` people, each group drawn its own size.""" + + people = int(rng.integers(1, max_people + 1)) + ids = {schema.person_entity: _sweep_ids(rng, people)} + memberships: dict[str, list[int]] = {} + for group in schema.group_entities: + count = int(rng.integers(1, people + 1)) + group_ids = sorted(_sweep_ids(rng, count)) + # The first ``count`` people of a shuffle take one group each, so every + # group has a member; the rest join any. + order = rng.permutation(people) + place = np.empty(people, dtype=np.int64) + place[order[:count]] = np.arange(count) + place[order[count:]] = rng.integers(0, count, size=people - count) + ids[group] = group_ids + memberships[group] = [group_ids[index] for index in place] + return SweepFrame(schema, ids, memberships) + + +def _drawn_column(rng: np.random.Generator, values: list[int]): + dtypes = [dtype for dtype in ID_DTYPES if holds(dtype, values)] + return typed(values, dtypes[int(rng.integers(0, len(dtypes)))]) + + +def sweep_tables( + frame: SweepFrame, rng: np.random.Generator +) -> dict[str, pd.DataFrame]: + """``frame``'s tables, each id and membership column of a dtype drawn + from those in ``ID_DTYPES`` that hold its values, beside one made-up + value column per table.""" + + schema = frame.schema + person_ids = frame.ids[schema.person_entity] + person: dict[str, object] = { + schema.person_id_column: _drawn_column(rng, person_ids) + } + for group in schema.group_entities: + person[schema.membership_column(group)] = _drawn_column( + rng, frame.memberships[group] + ) + person[f"{schema.person_entity}_value"] = np.arange(len(person_ids)) + 0.5 + tables = {schema.person_entity: pd.DataFrame(person)} + for group in schema.group_entities: + group_ids = frame.ids[group] + tables[group] = pd.DataFrame( + { + schema.id_column(group): _drawn_column(rng, group_ids), + f"{group}_value": np.arange(len(group_ids)) + 0.25, + } + ) + return tables + + +# --------------------------------------------------------------------------- +# UK frames +# --------------------------------------------------------------------------- + + +def _uk_names(): + """The UK adapter's schema and batch column, imported when a UK frame is + built so that importing this module loads no UK module.""" + + from microcosm.frame.adapters.policyengine_uk import UK_SCHEMA + from microcosm.frame.adapters.policyengine_uk_population import ( + UK_SPINE_ENGINE_BATCH_COLUMN, + ) + + return UK_SCHEMA, UK_SPINE_ENGINE_BATCH_COLUMN + + +def uk_frame( + *, + household_ids: Sequence[int], + benunit_ids: Sequence[int], + people: Sequence[tuple[int, int, int]], + person_columns: Mapping[str, Sequence[object]] | None = None, + benunit_columns: Mapping[str, Sequence[object]] | None = None, + household_columns: Mapping[str, Sequence[object]] | None = None, + household_weights: Sequence[float] | None = None, +) -> Frame: + """A Frame of the UK adapter's schema, its ids int64 as the spine's are. + + Households and benefit units take their ids in the order given (ascending, + as a Frame keeps them); ``people`` are (person id, benefit unit id, + household id) rows in the order given. Each ``*_columns`` mapping adds + columns in its table's row order; a NumPy or pandas array keeps its dtype. + Households weigh 100 each unless ``household_weights`` are given. + """ + + schema, _batch_column = _uk_names() + person = pd.DataFrame( + { + "person_id": np.asarray([row[0] for row in people], dtype=np.int64), + "person_benunit_id": np.asarray([row[1] for row in people], dtype=np.int64), + "person_household_id": np.asarray( + [row[2] for row in people], dtype=np.int64 + ), + } + ) + benunit = pd.DataFrame({"benunit_id": np.asarray(benunit_ids, dtype=np.int64)}) + household = pd.DataFrame( + {"household_id": np.asarray(household_ids, dtype=np.int64)} + ) + for table, columns in ( + (person, person_columns), + (benunit, benunit_columns), + (household, household_columns), + ): + for name, values in (columns or {}).items(): + # A NumPy or pandas array keeps its dtype; anything else is a list. + keeps = isinstance(values, np.ndarray | pd.api.extensions.ExtensionArray) + table[name] = values if keeps else list(values) + weights = ( + np.full(len(household), 100.0) + if household_weights is None + else np.asarray(household_weights, dtype=np.float64) + ) + return Frame( + tables={"person": person, "benunit": benunit, "household": household}, + schema=schema, + weights={"household": Weights(values=weights, kind=WeightKind.DESIGN)}, + ) + + +def clone_layout_frame( + clones: int = 3, + *, + shift: int = SHIFT, + shifted: Sequence[str] = ("person", "benunit", "household"), +) -> Frame: + """The UK engine-free adapter tests' clone layout, shifted. + + Two households per clone and one person each, as ``_clone_frame`` lays + them out (household and benefit-unit ids 1 to n, people listed in reverse + with ids ten times their household's), without the clone column. Every id + of an entity in ``shifted`` is moved by ``shift``, so, with the default, + the ids pass 2**31 and stay apart once stored as int32, as K = 1's SPI + copies do. + """ + + count = 2 * clones + ids = np.arange(1, count + 1, dtype=np.int64) + + def moved(entity: str, values: np.ndarray) -> list[int]: + return [int(value) + (shift if entity in shifted else 0) for value in values] + + reverse = ids[::-1] + people = list( + zip( + moved("person", reverse * 10), + moved("benunit", reverse), + moved("household", reverse), + strict=True, + ) + ) + return uk_frame( + household_ids=moved("household", ids), + benunit_ids=moved("benunit", ids), + people=people, + person_columns={"age": [40.0] * count}, + ) + + +def copy_layout_frame( + *, + person_columns: Mapping[str, Sequence[object]] | None = None, + benunit_columns: Mapping[str, Sequence[object]] | None = None, + household_columns: Mapping[str, Sequence[object]] | None = None, +) -> Frame: + """A UK frame whose every entity's ids are laid out as copy ids are. + + Households :data:`COPY_IDS` with :data:`LAYOUT_MEMBERS` people each (all + different), one benefit unit each (:data:`COPY_GROUP_IDS`), people + :data:`COPY_PERSON_IDS` out of id order. The copies' ids pass 2**31; every + id stays apart once stored as int32, and the ids convert out of int64 + order. People carry ``age`` (two adults, then children); extra columns go + in row order. + """ + + rows = _people(COPY_PERSON_IDS, LAYOUT_MEMBERS) + people = [(row[0], COPY_GROUP_IDS[row[1]], COPY_IDS[row[1]]) for row in rows] + columns = {"age": [_AGES[row[2]] for row in rows], **(person_columns or {})} + return uk_frame( + household_ids=COPY_IDS, + benunit_ids=COPY_GROUP_IDS, + people=people, + person_columns=columns, + benunit_columns=benunit_columns, + household_columns=household_columns, + household_weights=[100.0 + 10.0 * index for index in range(len(COPY_IDS))], + ) + + +def colliding_household_frame( + household_ids: Sequence[int] = COLLIDING_HOUSEHOLD_IDS, + *, + person_columns: Mapping[str, Sequence[object]] | None = None, + benunit_columns: Mapping[str, Sequence[object]] | None = None, + household_columns: Mapping[str, Sequence[object]] | None = None, +) -> Frame: + """A UK frame of households 7, 9 and 7 + 2**32, or ``household_ids``. + + Pass ``(1, 2, 3)`` for the same frame renumbered by hand. The households + hold two, one and three people, one benefit unit each; benefit units and + people fit int32, and people are listed out of id order. People carry + ``age``; extra columns go in row order (people as listed: the third + household's adult, the first's adult, the second's, the third's two + children, the first's child). + """ + + first, second, third = household_ids + people = [ + (34, 23, third), + (31, 21, first), + (33, 22, second), + (35, 23, third), + (36, 23, third), + (32, 21, first), + ] + columns = { + "age": [38.0, 40.0, 70.0, 12.0, 9.0, 6.0], + **(person_columns or {}), + } + return uk_frame( + household_ids=household_ids, + benunit_ids=(21, 22, 23), + people=people, + person_columns=columns, + benunit_columns=benunit_columns, + household_columns=household_columns, + household_weights=(110.0, 90.0, 130.0), + ) + + +def colliding_clones_frame( + *, + person_columns: Mapping[str, Sequence[object]] | None = None, + benunit_columns: Mapping[str, Sequence[object]] | None = None, + household_columns: Mapping[str, Sequence[object]] | None = None, +) -> Frame: + """A UK frame of two clones whose households store together. + + The households carry the clone column. Clone 0 holds households 7 and + 7 + 2**32 and clone 1 the same plus 10**9, so within each clone both + household ids store as one int32 value. The first household of each clone + holds an adult and a child, the second one adult, each household one + benefit unit. Benefit-unit and person ids pass 2**31 and stay apart once + stored, clone 1's being clone 0's plus 10**9. People are listed out of id + order with the clones interleaved, and carry ``age`` and the + ``months_since_last_birthday`` the spine's batched pass requires. + """ + + _schema, batch_column = _uk_names() + households, clones, benunits, people, ages, months = [], [], [], [], [], [] + for clone, (paired, wrapped_pair) in COLLIDING_CLONE_HOUSEHOLD_IDS.items(): + offset = clone * CLONE_OFFSET + first_unit, second_unit = 11 + 10**11 + offset, 11 + 10**12 + offset + households += [paired, wrapped_pair] + clones += [clone, clone] + benunits += [first_unit, second_unit] + people += [ + (3 + 10**11 + offset, first_unit, paired), + (3 + 10**12 + offset, first_unit, paired), + (5 + 10**11 + offset, second_unit, wrapped_pair), + ] + ages += [40.0, 9.0, 35.0] + months += [4.0 + clone, 10.0 - clone, 7.0 + 3 * clone] + household_order = np.argsort(households) + benunit_order = np.argsort(benunits) + # Clone 1's second household, clone 0's child, clone 1's adult, and so on. + rows = (5, 1, 3, 2, 4, 0) + return uk_frame( + household_ids=[households[index] for index in household_order], + benunit_ids=[benunits[index] for index in benunit_order], + people=[people[row] for row in rows], + person_columns={ + "age": [ages[row] for row in rows], + "months_since_last_birthday": [months[row] for row in rows], + **(person_columns or {}), + }, + benunit_columns=benunit_columns, + household_columns={ + batch_column: np.asarray( + [clones[index] for index in household_order], dtype=np.int64 + ), + **(household_columns or {}), + }, + household_weights=(100.0, 120.0, 140.0, 160.0), + ) + + +def relabelled(frame: Frame, entity: str, ids: Sequence[int]) -> Frame: + """``frame`` renumbered by hand: ``entity``'s ids replaced row for row by + ``ids``, and each membership of it mapped to its group's new id through a + dict, independently of the shared module's ranks. A group's new ids must + be ascending, as a Frame keeps them.""" + + schema = frame.schema + tables = {name: frame.table(name).copy() for name in frame.entities} + column = schema.entity_id_column(entity) + old = [int(value) for value in tables[entity][column]] + new = dict(zip(old, (int(value) for value in ids), strict=True)) + tables[entity][column] = np.asarray([new[value] for value in old], dtype=np.int64) + if entity != schema.person_entity: + membership = schema.membership_column(entity) + person = tables[schema.person_entity] + person[membership] = np.asarray( + [new[int(value)] for value in person[membership]], dtype=np.int64 + ) + return Frame( + tables, + schema, + {name: frame.weights_for(name) for name in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) From 26ba2a2048c2defb779c4928e47886777484c1ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:04:44 +0000 Subject: [PATCH 069/108] Refuse a dense build on a spine that does not assign areas (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dense line now refuses, before it pins an input or composes a graph, any spine that is not geography-first. A geography-first spine makes and keys its geographic clones; draws each clone's area, reads its local authority and fuel-relief flag off it and sets its BRMA, Great Britain's from the area; gives SPI copies and band donors areas of their own; re-draws Great Britain's private rents off BRMA rent lists; and localises England and Wales main-residence values inside the WAS chain. A bound spine's build sidecar also records how its area draw was bound. No spine built from this code is geography-first yet, so the dense line refuses every spine until the geography declarations land. The national line and the row-wise dataset tool are unchanged. Why. María's ruling of 10 Nov (decision 4) holds dense and national builds back until (c) re-bases them. C-21 re-bases the dense line in three commits, refusal first: this one, then the retirement of its property, BRMA, private-rent and fuel-relief nodes, then the dense line on the bound spine's clones, areas and sampling. With the refusal first, no commit in between can build a dense dataset on a half re-based line. - geography_first_roster (new) holds names only and imports nothing first-party: GEOGRAPHY_FIRST_STAGES (geographic_expand, geographic_identity, atomic_area_assign, local_authority, rfdr, area_brma, spi_copy_areas, private_rent), GEOGRAPHY_FIRST_OPERATIONS (was_wealth's localise_main_residence_value, in a read-only map) and SIDECAR_GEOGRAPHY_KEYS, the keys of the sidecar's geography block (assignment, identity, definition_sha256, support_pins, stream, register). It restates the stage names the transforms define: having geographic_expand and geographic_identity import them would re-key those live stages. A test holds the two equal. - spine_posture (new): roster_failures names, in the leaf's order, each stage, operation and block key a roster lacks. A roster that is not a list of names fails as a whole, extra names never fail, and only the block's keys are read, never their values. require_geography_first_spine refuses in one sentence that names what is missing and nothing else. stage_roster reads declared stages' names and operation kinds, and spec_roster a spec's. - full_build_cli: prepare_full_build runs the check right after the engine freeze (microcosm#1095). With --input-h5 it reads the build sidecar's stages, operations and geography block, from --input-sidecar where one is named, before the ladder or the spine is pinned; an absent or unreadable sidecar is refused in load_bound_spine_checkpoint's words. With --spine-request it reads the stage names and operation kinds of the packaged UK spec, whose whole roster every request runs, without parsing the request or composing its spine. The check then runs again on the roster the build binds: the sidecar load_bound_spine_checkpoint authenticates, and the stages prepare_uk_spine_execution composes. So a sidecar changed between the two reads, or a composed spine that differs from the spec, is refused too. What moves. Node keys: none. Of the 101 kernels the UK registries register, none hashes full_build_cli, spine_posture or geography_first_roster, registering them imports neither full_build_cli nor spine_posture, and no shared module changes. Values: none. No spine module and no dense compute module changes, so K = 1 spine values cannot move. The dense expansion already refused, at any clone count, a spine carrying geographic_expand's clone indices, as every spine built since (b) does, but only once the graph reached it, so a dense dry run, with or without --candidate-clone-counts, could still plan on such a spine. Dry runs and builds now both stop at preparation. The check also refuses a spine without clone indices, one built before (b), which the dense line could still bind. The atomic-assignment measurement tool's graph route runs the dense preparation, so it refuses too until a later (c) commit re-points it at the spine's own draw. A refused build records its attempt as failed, with the refusal in failure.json, and runs no graph. No declaration, generated file, gates.json or schema changes, and H2 is not regenerated. Tests (invented rosters, the packaged spec, and the spine build's own sidecar writer over a synthetic frame; no built spine's sidecar): - test_uk_spine_posture (new, 10 tests): the leaf's names, its read-only map, and that it imports nothing first-party; its stage names equal the transforms' constants; the packaged roster, and the sidecar spine_build._build_sidecar writes for it, are refused naming the six area stages and the localiser, the sidecar also the block; stage_roster and spec_roster read names and kinds in order; each missing name fails alone, in the leaf's order; unreadable rosters fail by name; a geography-first stand-in passes, with extra names too; no refusal carries a figure. - test_uk_full_build_cli (17 new cases): on both routes, recording stand-ins show that nothing is pinned, read or composed before the refusal, with the real request-roster reader running under them, and that a geography-first stand-in goes on to pin the ladder; a sidecar block that lacks any one key is refused naming that key; an absent, malformed or non-object sidecar is refused word for word as the bound loader refuses it; a roster that passes the first check is refused, before the spine is used, when the authenticated sidecar or the composed spine lacks the area stages; the request roster equals the packaged spec's and the spine build's stage names; through main, a refused build exits 1 with failure.json and a failed Logbook row and runs no graph, and a dry run raises the refusal. - test_uk_full_build_preparation and test_uk_rowwise_candidate run the real preparation over stand-in sidecars, which now carry a geography-first roster from test support (geography_first_sidecar_fields); one provenance assertion follows. Checks: test_uk_spine_posture, test_uk_full_build_cli, test_uk_full_build_preparation, test_uk_rowwise_candidate and test_uk_hmrc_uprating_engine_pins (254 passed); from test_uk_graph.py, the kernel census, annuity-column reader, household-type check and battery-engine tests (4 passed); with either second check removed, its test fails; ruff check and ruff format --check on the eight edited Python files; ci_test_plan.py verify. Nothing here builds a Simulation, so the engine slot was not used. Co-Authored-By: Claude Opus 5.5 --- ...e-needs-a-geography-first-spine.changed.md | 14 + .../build/uk_runtime/full_build_cli.py | 76 ++- .../uk_runtime/geography_first_roster.py | 51 +++ .../build/uk_runtime/spine_posture.py | 150 ++++++ .../engine_free/uk/test_uk_full_build_cli.py | 432 ++++++++++++++++++ .../uk/test_uk_full_build_preparation.py | 11 +- .../uk/test_uk_rowwise_candidate.py | 6 +- .../engine_free/uk/test_uk_spine_posture.py | 337 ++++++++++++++ .../microcosm_build/uk_full_build_cli.py | 57 ++- 9 files changed, 1126 insertions(+), 8 deletions(-) create mode 100644 changelog.d/931-dense-needs-a-geography-first-spine.changed.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/geography_first_roster.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/spine_posture.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_spine_posture.py diff --git a/changelog.d/931-dense-needs-a-geography-first-spine.changed.md b/changelog.d/931-dense-needs-a-geography-first-spine.changed.md new file mode 100644 index 0000000000..7921d14cf7 --- /dev/null +++ b/changelog.d/931-dense-needs-a-geography-first-spine.changed.md @@ -0,0 +1,14 @@ +The dense line refuses a spine that does not assign areas, before it pins or reads any other input (#931; María's ruling of 10 Nov, decision 4). Later commits of #931 group (c) move the dense line onto the spine's own areas; refusing first means none of them can build a dense dataset on a half-moved line. `full_build_cli.prepare_full_build` now checks the spine's roster first: after the engine freeze, and before it pins an input, reads the spine, authenticates the checkpoint or composes a graph. It checks again the roster the build then binds. No spine built from this code passes yet. The check passes once the spine declares its area stages and the main-residence localiser, and a bound spine's sidecar records the geography block. + +- **The geography-first names** (`microcosm.build.uk_runtime.geography_first_roster`, new). A names-only leaf that imports nothing first-party: `GEOGRAPHY_FIRST_STAGES` (`geographic_expand`, `geographic_identity`, `atomic_area_assign`, `local_authority`, `rfdr`, `area_brma`, `spi_copy_areas` and `private_rent`), `GEOGRAPHY_FIRST_OPERATIONS` (`was_wealth`'s `localise_main_residence_value`) and `SIDECAR_GEOGRAPHY_KEYS`, the keys of the build sidecar's `geography` block (`assignment`, `identity`, `definition_sha256`, `support_pins`, `stream` and `register`). Its stage names are the ones the stages' transforms declare, and a test holds them together. +- **The check** (`microcosm.build.uk_runtime.spine_posture`, new). `roster_failures(stages, operations, geography=None, *, from_sidecar=True)` names, in the leaf's order, every stage, operation and block key a roster lacks. A roster that is not names fails as a whole (`stage roster`, `operation roster`), an operation counts only in a stage the roster ran, and extra names never fail. Only the block's keys are read, never their values. `require_geography_first_spine` refuses a roster that lacks anything, in one sentence that names what it lacks and nothing else: no count, id or value. `stage_roster` reads declared stages' names and operation kinds, and `spec_roster` a spec's. +- **Both routes.** + - With `--input-h5` the check reads the build sidecar's `stages`, `operations` and `geography`, from `--input-sidecar` where it is named, before the ladder or the spine is pinned. An absent or unreadable sidecar is refused in `load_bound_spine_checkpoint`'s own words. The checkpoint is still authenticated after the check, and the sidecar it authenticates, the one the build binds, is checked again, so one changed since the first read is refused too. + - With `--spine-request` it reads the stage names and operation kinds of the packaged UK spec, whose whole roster every request runs, before the request is parsed or its spine composed. Its spine runs inside the dense build's own graph, so it has no block to read. The spine the request composes is checked again, on its own stages, before the build reads it. + - A dry run, with or without `--candidate-clone-counts`, is refused alike. The national line and the row-wise dataset tool (`tools/build_uk_rowwise_dataset.py`) are unchanged. +- **What moves.** No line builds differently: the dense expansion already refused, at any clone count, a spine that carries the geographic clone indices, as every spine built since #931 group (b) does. The check also refuses a spine without them, one built before group (b), which the dense line could still bind until now. The expansion refused only once the graph reached it, so a dense dry run could still plan on a clone-first spine; dry runs and builds now both stop at preparation. A refused build records its attempt as failed, with the refusal as its message. `tools/measure_uk_atomic_assignment.py`'s graph route runs the dense preparation, so it is refused the same way. No node hashes `full_build_cli` or the two new modules, so no node key moves. No declaration, generated file, `gates.json` or schema changes. +- **Tests** (invented rosters, rosters read off the packaged spec, and the spine build's own sidecar writer over a synthetic frame; no built spine's sidecar). + - The packaged roster, and the sidecar the spine build's writer records for it, are refused naming the six area stages and the localiser, and the sidecar also the block. + - Each missing name fails alone, in the leaf's order; unreadable rosters fail by name; a geography-first stand-in passes, with extra names too; no refusal carries a figure. + - Through `prepare_full_build` on both routes, recording stand-ins, under which the real request-roster reader runs, show the refusal comes before any input is pinned or read and before any graph is composed, and that a geography-first stand-in goes on to pin its first input. A sidecar block that lacks any one key is refused naming that key; an unreadable sidecar is refused word for word as the bound loader refuses it. A roster that passes the first check but not on the authenticated sidecar or the composed spine is refused before the spine is used. Through `main`, a refused build records a failed attempt without running a graph. + - The stand-in sidecars of the preparation tests and of the dense dry-run test carry a geography-first roster (`geography_first_sidecar_fields` in `test_support/microcosm_build/uk_full_build_cli.py`). diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 5e32ba43ff..4fbbb8655f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -14,7 +14,10 @@ :mod:`microcosm.build.uk_runtime.national_role`) materialised from the graph's stored artifacts. Both lines are built here through the graph. The role supplies every unset solve default and refuses the other role's flags -through :mod:`microcosm.build.uk_runtime.rowwise_cli`. +through :mod:`microcosm.build.uk_runtime.rowwise_cli`. The dense line builds +only on a geography-first spine: :func:`prepare_full_build` refuses any other +before it pins an input or composes a graph +(:mod:`microcosm.build.uk_runtime.spine_posture`, microcosm#931). ``tools/build_uk_rowwise_candidate.py`` and ``tools/build_uk_full.py`` are stubs over :func:`main`, so every UK line is built by this one command. @@ -257,6 +260,7 @@ ) from .size_checkpoint import uk_size_checkpoint_identity from .solve_scope import UK_GATE_ONLY_RESOURCES, UK_POST_SOLVE_RESOURCES +from .spine_posture import require_geography_first_spine, spec_roster, stage_roster from .staging import UK_STAGED_DATASET_REPOSITORY, UK_STAGING_REPOSITORY from .weight_year import uk_dataset_year_frame @@ -1305,6 +1309,63 @@ def _require_gate_kernel_completed(manifest, node_id: str) -> None: ) +def _read_spine_sidecar(path: Path) -> dict: + """A bound spine's build sidecar as a JSON object. + + The refusals are ``load_bound_spine_checkpoint``'s, word for word, so a + sidecar that cannot be read is refused alike whichever reader meets it + first. + """ + if not path.is_file(): + raise ValueError(f"input H5 build sidecar absent: {path}") + try: + sidecar = json.loads(path.read_bytes()) + except (json.JSONDecodeError, UnicodeError) as exc: + raise ValueError(f"input H5 build sidecar is invalid JSON: {path}") from exc + if not isinstance(sidecar, dict): + raise ValueError(f"input H5 build sidecar must be a JSON object: {path}") + return sidecar + + +def _spine_request_roster() -> tuple[tuple[str, ...], dict[str, tuple[str, ...]]]: + """The stage names and operation kinds a spine request runs. + + Every request runs the packaged UK spec's whole roster: no spine argument + selects stages, and the spine build derives its roster from that spec, + refusing a spine graph that disagrees with it. So the roster is read off + the spec, without parsing the request or composing its spine. + """ + from microcosm.build.country_spec import load_country_spec + + return spec_roster(load_country_spec("uk")) + + +def _require_geography_first_spine(args: argparse.Namespace) -> None: + """Refuse a spine that does not assign areas, before any input is pinned. + + The dense line builds only on a geography-first spine (microcosm#931; + :mod:`~microcosm.build.uk_runtime.spine_posture`). A bound spine + (``--input-h5``) is read off its build sidecar: the stages it ran, their + operation kinds and its ``geography`` block. A spine request's stage names + and operation kinds are read off the packaged spec it runs + (:func:`_spine_request_roster`); its spine runs inside the dense build's + own graph, so it has no block to read. The check only refuses: it reads + the sidecar before the spine is pinned, the bound checkpoint is still + authenticated after it, and :func:`prepare_full_build` checks the roster + it then binds again, the authenticated sidecar's or the composed spine's. + """ + if args.input_h5 is not None: + sidecar = _read_spine_sidecar( + Path(args.input_sidecar or args.input_h5.with_suffix(".build.json")) + ) + require_geography_first_spine( + sidecar.get("stages"), sidecar.get("operations"), sidecar.get("geography") + ) + else: + stages, operations = _spine_request_roster() + require_geography_first_spine(stages, operations, from_sidecar=False) + + def prepare_full_build( args: argparse.Namespace, *, telemetry=None, attempt: dict | None = None ) -> PreparedUKFullBuild: @@ -1320,6 +1381,9 @@ def prepare_full_build( # The engine freeze, before any input is read (microcosm#1095). assert_installed_engine_is_pinned() + # The spine's posture, before any input is pinned or any graph composed: + # the dense line builds only on a spine that assigns areas (microcosm#931). + _require_geography_first_spine(args) posture = posture_of(args) release = load_uk_frs_release() pins = {"ladder": _pin(args.ladder, args.ladder_sha256)} @@ -1347,6 +1411,11 @@ def prepare_full_build( gate_report_path=gates_path, input_sha256=pins["dataset"]["sha256"], ) + # The authenticated sidecar is the one the build binds, so its roster + # is checked again: one changed since the early read is refused too. + require_geography_first_spine( + sidecar.get("stages"), sidecar.get("operations"), sidecar.get("geography") + ) spine = bound_spine_graph(frame) endpoint = "uk.full.spine_checkpoint" weight_kind = frame.weights_for("household").kind.value @@ -1384,6 +1453,11 @@ def prepare_full_build( if args.release_candidate: raw.release_candidate = True prepared = prepare_uk_spine_execution(raw) + # The composed spine is the one the build runs, so its own roster is + # checked again: the early check read the packaged spec's. + require_geography_first_spine( + *stage_roster(prepared.stages), from_sidecar=False + ) spine, kernels = prepared.graph, prepared.kernels sources.update(prepared.sources) endpoint = uk_spine_endpoint(spine).population diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_first_roster.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_first_roster.py new file mode 100644 index 0000000000..9030c8b27c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_first_roster.py @@ -0,0 +1,51 @@ +"""The names that make a UK spine geography-first (microcosm#931). + +A geography-first spine runs every stage ``GEOGRAPHY_FIRST_STAGES`` names and +``was_wealth``'s ``localise_main_residence_value`` operation, and its build +sidecar records how its area draw was bound in a ``geography`` block under +``SIDECAR_GEOGRAPHY_KEYS``. The dense line builds only on such a spine +(:mod:`~microcosm.build.uk_runtime.spine_posture`); the spine build's sidecar +writer is to write the block under these keys, and a test holds the stage +names to their transforms' own. Names only, importing nothing first-party. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType + +__all__ = [ + "GEOGRAPHY_FIRST_OPERATIONS", + "GEOGRAPHY_FIRST_STAGES", + "SIDECAR_GEOGRAPHY_KEYS", +] + +#: The stages a geography-first spine runs, in the spine's order. +GEOGRAPHY_FIRST_STAGES: tuple[str, ...] = ( + "geographic_expand", + "geographic_identity", + "atomic_area_assign", + "local_authority", + "rfdr", + "area_brma", + "spi_copy_areas", + "private_rent", +) + +#: The operations a geography-first spine runs inside stages that are not its +#: own, by stage: the main-residence localiser runs inside the WAS chain. +GEOGRAPHY_FIRST_OPERATIONS: Mapping[str, tuple[str, ...]] = MappingProxyType( + {"was_wealth": ("localise_main_residence_value",)} +) + +#: The keys of the build sidecar's ``geography`` block: the assignment, the +#: identity column it draws on, the definition's digest, the support pins, the +#: draw stream and the register the pins are held to. +SIDECAR_GEOGRAPHY_KEYS: tuple[str, ...] = ( + "assignment", + "identity", + "definition_sha256", + "support_pins", + "stream", + "register", +) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_posture.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_posture.py new file mode 100644 index 0000000000..03d037909f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_posture.py @@ -0,0 +1,150 @@ +"""The spine a UK dense build stands on must be geography-first (microcosm#931). + +The dense line builds only on a spine that assigns each geographic clone its +area: one that runs every stage +:data:`~microcosm.build.uk_runtime.geography_first_roster.GEOGRAPHY_FIRST_STAGES` +names and every operation ``GEOGRAPHY_FIRST_OPERATIONS`` names, and, where a +build sidecar describes it, records the ``geography`` block under the keys +``SIDECAR_GEOGRAPHY_KEYS`` names. :func:`roster_failures` names what a roster +lacks; :func:`require_geography_first_spine` refuses a spine that lacks +anything. The dense line's preparation calls it right after the engine freeze, +before it pins an input or composes a graph, and again on the roster the build +then binds. + +The check reads names only: the stages a spine ran or will run, each stage's +operation kinds, and the keys of the sidecar's ``geography`` block, never their +values. Its texts give names only, never a count, an id or a value. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence + +from microcosm.build.uk_runtime.geography_first_roster import ( + GEOGRAPHY_FIRST_OPERATIONS, + GEOGRAPHY_FIRST_STAGES, + SIDECAR_GEOGRAPHY_KEYS, +) + +__all__ = [ + "require_geography_first_spine", + "roster_failures", + "spec_roster", + "stage_roster", +] + + +def stage_roster( + stages: Iterable[object], +) -> tuple[tuple[str, ...], dict[str, tuple[str, ...]]]: + """The names of declared ``stages``, in their order, and each one's + operation kinds: the names a build sidecar records under ``stages`` and + ``operations``.""" + + declared = tuple(stages) + names = tuple(stage.stage for stage in declared) + operations = { + stage.stage: tuple(operation.kind for operation in stage.operations) + for stage in declared + } + return names, operations + + +def spec_roster(spec) -> tuple[tuple[str, ...], dict[str, tuple[str, ...]]]: + """:func:`stage_roster` of the stages a country spec declares, in its + order.""" + + if spec.sources is None: + raise ValueError("The country spec declares no source stages.") + return stage_roster(spec.sources.stages) + + +def _names(value: object) -> tuple[str, ...] | None: + """``value`` as a sequence of names, or ``None`` if it is not one.""" + + if isinstance(value, str | bytes) or not isinstance(value, Sequence): + return None + if not all(isinstance(item, str) for item in value): + return None + return tuple(value) + + +def roster_failures( + stages: object, + operations: object, + geography: object = None, + *, + from_sidecar: bool = True, +) -> tuple[str, ...]: + """The geography-first names this roster lacks, in the leaf's order. + + ``stages`` and ``operations`` are a sidecar's fields of those names, or a + spine request's roster; of ``geography``, the sidecar's block, only the + keys are read. Failures read ``stage ``, ``operation in + ``, ``geography block`` and ``geography key ``; a roster that + is not names fails as ``stage roster`` or ``operation roster``, and a + stage whose kinds are not names holds none. An operation counts only in a + stage the roster ran; extra names never fail. With ``from_sidecar=False`` + no block is read, and passing one raises. + """ + + if not from_sidecar and geography is not None: + raise ValueError( + "A roster read without a sidecar has no geography block to check." + ) + failures: list[str] = [] + names = _names(stages) + if names is None: + failures.append("stage roster") + else: + ran = set(names) + failures.extend( + f"stage {stage}" for stage in GEOGRAPHY_FIRST_STAGES if stage not in ran + ) + if not isinstance(operations, Mapping): + failures.append("operation roster") + else: + for stage, required in GEOGRAPHY_FIRST_OPERATIONS.items(): + kinds = _names(operations.get(stage)) if stage in ran else None + failures.extend( + f"operation {kind} in {stage}" + for kind in required + if kind not in (kinds or ()) + ) + if from_sidecar: + if not isinstance(geography, Mapping): + failures.append("geography block") + else: + failures.extend( + f"geography key {key}" + for key in SIDECAR_GEOGRAPHY_KEYS + if key not in geography + ) + return tuple(failures) + + +def require_geography_first_spine( + stages: object, + operations: object, + geography: object = None, + *, + from_sidecar: bool = True, +) -> None: + """Refuse a spine whose roster lacks any geography-first name. + + The arguments are :func:`roster_failures`'. The refusal is one sentence + that names what the roster lacks and nothing else. + """ + + failures = roster_failures(stages, operations, geography, from_sidecar=from_sidecar) + if failures: + where = ( + "the bound spine's build sidecar" + if from_sidecar + else "the spine request's stage roster" + ) + raise ValueError( + "The dense line builds only on a geography-first spine, one that " + "assigns every geographic clone its area (microcosm#931); " + f"{where} lacks: {'; '.join(failures)}." + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index bea44698d3..30d1496c29 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -410,6 +410,438 @@ def test_candidate_clone_counts_are_dry_run_only(tmp_path): ) +# --------------------------------------------------------------------------- +# microcosm#931: the dense line builds only on a geography-first spine +# --------------------------------------------------------------------------- + + +class _StandInStopError(Exception): + """A recording stand-in's stop: the preparation got past the posture check.""" + + +def _record_preparation(monkeypatch) -> list[str]: + """Recording stand-ins for every preparation step after the posture check + that reads an input or composes a graph. Each records its name and stops + the preparation, so the record holds the first step the check let through, + or nothing. The engine freeze, which runs before the check, stands in.""" + from microcosm.build.uk_runtime import calibration_run, hmrc_uprating, spine_build + from microcosm.build.uk_runtime import graph as uk_graph + + calls: list[str] = [] + + def stand_in(name): + def record(*args, **kwargs): + calls.append(name) + raise _StandInStopError(name) + + return record + + monkeypatch.setattr( + hmrc_uprating, "assert_installed_engine_is_pinned", lambda: "fixture" + ) + for owner, name in ( + (cli, "_pin"), + (cli, "load_uk_national_frame"), + (cli, "bound_spine_graph"), + (cli, "uk_full_graph"), + (cli, "append_uk_full_gate_nodes"), + (cli, "compile_graph"), + (cli, "_prepare_geography"), + (calibration_run, "load_bound_spine_checkpoint"), + (spine_build, "parse_uk_spine_args"), + (spine_build, "prepare_uk_spine_execution"), + (spine_build, "uk_spine_graph"), + # A reader that imports the spine graph's composer late meets it here. + (uk_graph, "uk_spine_graph"), + ): + monkeypatch.setattr(owner, name, stand_in(name)) + return calls + + +def _write_sidecar(tmp_path, roster) -> None: + """A stand-in build sidecar beside ``arguments``' stand-in spine.""" + (tmp_path / "spine.build.json").write_text(json.dumps(roster)) + + +def _packaged_sidecar_roster() -> dict: + """The packaged spec's stages and kinds as sidecar fields (lists), with no + geography block: the names a spine built on this code records.""" + stages, operations = cli._spine_request_roster() + return { + "stages": list(stages), + "operations": {stage: list(kinds) for stage, kinds in operations.items()}, + } + + +#: A spine built before the area stages, cut to what the posture check reads: +#: it makes and keys its clones, frs_brma draws the BRMAs, and was_wealth runs +#: without the main-residence localiser. +_PRE_GEOGRAPHY_ROSTER = { + "stages": [ + "frs_spine", + "geographic_expand", + "geographic_identity", + "frs_brma", + "was_wealth", + ], + "operations": { + "frs_spine": ["read_tables"], + "geographic_expand": ["expand_geographic_clones"], + "geographic_identity": ["key_households_by_clone_path"], + "frs_brma": [ + "materialize_rules_engine_predictors", + "sample_categorical_from_count_table", + ], + "was_wealth": ["derive", "fit_weighted_qrf_chain"], + }, +} + + +def _posture_refusal(roster, *, from_sidecar=True) -> str: + """The posture check's own refusal of ``roster``.""" + from microcosm.build.uk_runtime.spine_posture import require_geography_first_spine + + with pytest.raises(ValueError) as refused: + require_geography_first_spine( + roster["stages"], + roster["operations"], + roster.get("geography"), + from_sidecar=from_sidecar, + ) + return str(refused.value) + + +def _request_arguments(tmp_path): + """A dense request over a spine request, with staging disabled.""" + request = tmp_path / "request.json" + request.write_text(json.dumps(["--smoke"])) + ladder = tmp_path / "ladder.npz" + return cli.parse_args( + [ + "--release-role", + "dense", + "--spine-request", + str(request), + "--ladder", + str(ladder), + "--ladder-sha256", + _placeholder(ladder, b"ladder stand-in"), + "--ledger-facts", + str(tmp_path / "ledger"), + "--ledger-facts-sha256", + PIN, + "--ledger-manifest-sha256", + PIN, + "--out", + str(tmp_path / "out"), + "--no-staging", + *SUPPORT_ARGUMENTS, + ] + ) + + +def _without_geography(roster) -> dict: + return {key: value for key, value in roster.items() if key != "geography"} + + +@pytest.mark.parametrize( + "roster", + [ + pytest.param(_packaged_sidecar_roster, id="the packaged roster"), + pytest.param(lambda: _PRE_GEOGRAPHY_ROSTER, id="a pre-geography roster"), + pytest.param( + lambda: _without_geography(geography_first_sidecar_fields("frs_spine")), + id="every stage but no geography block", + ), + ], +) +def test_a_bound_spine_that_does_not_assign_areas_is_refused_before_any_input_is_pinned( + tmp_path, monkeypatch, roster +): + """``--input-h5``: the sidecar's roster is checked before the ladder or the + spine is pinned, the frame read, the checkpoint authenticated or any graph + composed (microcosm#931), and the refusal is the posture check's own, + naming every missing name and nothing else.""" + fields = roster() + _write_sidecar(tmp_path, fields) + calls = _record_preparation(monkeypatch) + with pytest.raises(ValueError) as refused: + cli.prepare_full_build(arguments(tmp_path)) + message = str(refused.value) + assert message == _posture_refusal(fields) + assert "the bound spine's build sidecar lacks:" in message + assert message.endswith("geography block.") + assert calls == [] + + +@pytest.mark.parametrize("missing", SIDECAR_GEOGRAPHY_KEYS) +def test_a_bound_spine_whose_geography_block_lacks_a_key_is_refused( + tmp_path, monkeypatch, missing +): + """A sidecar that records every geography-first stage and operation but + whose block lacks one key is refused through the preparation, naming that + key alone, before anything is pinned.""" + roster = geography_first_sidecar_fields("frs_spine") + del roster["geography"][missing] + _write_sidecar(tmp_path, roster) + calls = _record_preparation(monkeypatch) + with pytest.raises(ValueError) as refused: + cli.prepare_full_build(arguments(tmp_path)) + assert str(refused.value) == _posture_refusal(roster) + assert str(refused.value).endswith(f"lacks: geography key {missing}.") + assert calls == [] + + +def test_a_bound_spine_whose_sidecar_cannot_be_read_is_refused_before_anything_is_pinned( + tmp_path, monkeypatch +): + """An absent, malformed or non-object sidecar is refused before anything + is pinned, word for word as the bound loader refuses it, which it does + before it reads the frame.""" + from microcosm.build.uk_runtime import calibration_run + + load_bound_spine_checkpoint = calibration_run.load_bound_spine_checkpoint + calls = _record_preparation(monkeypatch) + args = arguments(tmp_path) + sidecar = tmp_path / "spine.build.json" + for text, needle in ( + (None, "absent"), + ("{", "is invalid JSON"), + ("[]", "must be a JSON object"), + ): + if text is not None: + sidecar.write_text(text) + expected = f"input H5 build sidecar {needle}" + with pytest.raises(ValueError, match=expected) as refused: + cli.prepare_full_build(args) + with pytest.raises(ValueError) as loaded: + load_bound_spine_checkpoint(sidecar, None) + assert str(refused.value) == str(loaded.value) + assert calls == [] + + +def test_a_bound_spine_read_through_its_named_sidecar_goes_on_once_geography_first( + tmp_path, monkeypatch +): + """A geography-first sidecar passes, read from ``--input-sidecar`` where + one is named (no sidecar lies beside the stand-in spine then) and from + beside the spine otherwise, and the preparation goes on to pin its first + input, the ladder.""" + calls = _record_preparation(monkeypatch) + named = tmp_path / "named.build.json" + named.write_text(json.dumps(geography_first_sidecar_fields("frs_spine"))) + args = arguments(tmp_path, "--input-sidecar", str(named)) + with pytest.raises(_StandInStopError): + cli.prepare_full_build(args) + assert calls == ["_pin"] + # The same roster beside the spine, where no sidecar is named, passes too. + calls.clear() + _write_sidecar(tmp_path, geography_first_sidecar_fields("frs_spine")) + with pytest.raises(_StandInStopError): + cli.prepare_full_build(arguments(tmp_path)) + assert calls == ["_pin"] + + +def test_a_bound_spine_is_checked_again_on_the_sidecar_it_authenticates( + tmp_path, monkeypatch +): + """The sidecar the bound loader authenticates is the one the build binds, + so its roster is checked again: a spine whose sidecar passed the early read + but whose authenticated sidecar does not assign areas is refused before + its graph is made.""" + from microcosm.build.uk_runtime import calibration_run + + pin = cli._pin + calls = _record_preparation(monkeypatch) + _write_sidecar(tmp_path, geography_first_sidecar_fields("frs_spine")) + # Past the early check the inputs are pinned for real and the frame stands + # in; the loader authenticates a sidecar changed since the early read. + monkeypatch.setattr(cli, "_pin", pin) + monkeypatch.setattr(cli, "load_uk_national_frame", lambda path: (object(), None)) + monkeypatch.setattr( + cli, "require_uk_producer_isa_income_contract", lambda frame, *, label: None + ) + monkeypatch.setattr( + calibration_run, + "load_bound_spine_checkpoint", + lambda *args, **kwargs: json.loads(json.dumps(_PRE_GEOGRAPHY_ROSTER)), + ) + with pytest.raises(ValueError) as refused: + cli.prepare_full_build(arguments(tmp_path)) + assert str(refused.value) == _posture_refusal(_PRE_GEOGRAPHY_ROSTER) + assert calls == [] + + +def test_a_spine_request_that_does_not_assign_areas_is_refused_before_it_is_read( + tmp_path, monkeypatch +): + """``--spine-request``: the roster the request runs is checked, its stage + names and operation kinds only, before the request is parsed or its spine + composed; the request's spine has no sidecar block to read + (microcosm#931).""" + calls = _record_preparation(monkeypatch) + pre = _PRE_GEOGRAPHY_ROSTER + monkeypatch.setattr( + cli, + "_spine_request_roster", + lambda: (tuple(pre["stages"]), dict(pre["operations"])), + ) + args = _request_arguments(tmp_path) + with pytest.raises(ValueError) as refused: + cli.prepare_full_build(args) + message = str(refused.value) + assert message == _posture_refusal(pre, from_sidecar=False) + assert "the spine request's stage roster lacks:" in message + assert "geography block" not in message and "geography key" not in message + assert calls == [] + # A geography-first roster passes; the preparation goes on to pin the + # ladder, its first input, and has parsed no request. + first = _without_geography(geography_first_sidecar_fields("frs_spine")) + monkeypatch.setattr( + cli, + "_spine_request_roster", + lambda: (tuple(first["stages"]), dict(first["operations"])), + ) + with pytest.raises(_StandInStopError): + cli.prepare_full_build(args) + assert calls == ["_pin"] + + +def test_a_spine_request_reads_its_roster_without_composing_a_graph( + tmp_path, monkeypatch +): + """The real request-roster reader runs under the recording stand-ins, so a + reader that composed the spine graph to read its roster would be caught. + While the packaged roster lacks a geography-first name it is refused + before anything is pinned, read or composed; once it has them all, the + preparation goes on to pin the ladder.""" + from microcosm.build.country_spec import load_country_spec + from microcosm.build.uk_runtime.spine_posture import roster_failures, spec_roster + + stages, operations = spec_roster(load_country_spec("uk")) + calls = _record_preparation(monkeypatch) + with pytest.raises((ValueError, _StandInStopError)) as stopped: + cli.prepare_full_build(_request_arguments(tmp_path)) + if roster_failures(stages, operations, from_sidecar=False): + assert str(stopped.value) == _posture_refusal( + {"stages": stages, "operations": operations}, from_sidecar=False + ) + assert calls == [] + else: + assert calls == ["_pin"] + + +def test_a_spine_request_is_checked_again_on_the_spine_it_composes( + tmp_path, monkeypatch +): + """The spine a request composes is the one the build runs, so its own + roster is checked again: a composed spine that lacks the area stages is + refused before the build reads it, though the roster read early passed.""" + from microcosm.build.uk_runtime import graph as uk_graph + from microcosm.build.uk_runtime import spine_build + + pin = cli._pin + calls = _record_preparation(monkeypatch) + first = _without_geography(geography_first_sidecar_fields("frs_spine")) + monkeypatch.setattr( + cli, + "_spine_request_roster", + lambda: (tuple(first["stages"]), dict(first["operations"])), + ) + pre = _PRE_GEOGRAPHY_ROSTER + composed = SimpleNamespace( + graph=None, + kernels=None, + sources={}, + stages=tuple( + SimpleNamespace( + stage=name, + operations=tuple( + SimpleNamespace(kind=kind) for kind in pre["operations"][name] + ), + ) + for name in pre["stages"] + ), + ) + # Past the early check the ladder is pinned for real, and the request + # composes the stand-in spine. + monkeypatch.setattr(cli, "_pin", pin) + monkeypatch.setattr( + spine_build, + "parse_uk_spine_args", + lambda argv: SimpleNamespace(release_candidate=False), + ) + monkeypatch.setattr(spine_build, "prepare_uk_spine_execution", lambda raw: composed) + monkeypatch.setattr( + uk_graph, + "uk_spine_endpoint", + lambda graph: pytest.fail("the build read a spine its roster check missed"), + ) + with pytest.raises(ValueError) as refused: + cli.prepare_full_build(_request_arguments(tmp_path)) + assert str(refused.value) == _posture_refusal(pre, from_sidecar=False) + assert calls == [] + + +def test_a_spine_request_is_checked_on_the_packaged_roster_it_runs(): + """No spine argument selects stages: a request runs the packaged UK spec's + whole roster, the one the spine build derives and composes, so the check + reads that spec's stage names and each stage's operation kinds.""" + from microcosm.build.country_spec import load_country_spec + from microcosm.build.uk_runtime.spine_build import _uk_spine_stage_names + + spec = load_country_spec("uk") + stages, operations = cli._spine_request_roster() + assert stages == tuple(stage.stage for stage in spec.sources.stages) + assert set(stages) == set(_uk_spine_stage_names(spec)) + assert operations == { + stage.stage: tuple(operation.kind for operation in stage.operations) + for stage in spec.sources.stages + } + + +def test_a_dense_build_on_a_spine_without_areas_fails_before_any_graph_runs( + tmp_path, monkeypatch, capsys +): + """Through ``main``, the refusal closes the attempt failed, with the + refusal as its message and no graph composed or run; a dry run raises + it.""" + from microcosm.build.logbook import load_spool_rows + from microcosm.build.uk_runtime import hmrc_uprating + + monkeypatch.delenv("POPULACE_LOGBOOK_PREV_ROW_DIGEST", raising=False) + monkeypatch.setattr( + hmrc_uprating, "assert_installed_engine_is_pinned", lambda: "fixture" + ) + for name in ( + "run_graph", + "compile_graph", + "uk_full_graph", + "load_uk_national_frame", + ): + monkeypatch.setattr( + cli, + name, + lambda *a, _name=name, **k: pytest.fail(f"{_name} ran before the refusal"), + ) + _write_sidecar(tmp_path, _packaged_sidecar_roster()) + refusal = _posture_refusal(_packaged_sidecar_roster()) + args = arguments(tmp_path) + monkeypatch.setattr(cli, "parse_args", lambda argv: args) + assert cli.main([]) == 1 + assert refusal in capsys.readouterr().err + failure = json.loads((args.out / "failure.json").read_text()) + assert (failure["error_type"], failure["message"]) == ("ValueError", refusal) + rows = load_spool_rows(args.out / "logbook-spool") + assert len(rows) == 1 and rows[0].disposition == "failed" + dry = arguments(tmp_path, "--dry-run", "--out", str(tmp_path / "dry")) + monkeypatch.setattr(cli, "parse_args", lambda argv: dry) + with pytest.raises(ValueError) as refused: + cli.main([]) + assert str(refused.value) == refusal + assert not (tmp_path / "dry").exists() + + def test_source_sampling_cannot_be_reapplied_as_pool_sampling(): config = UKFullBuildConfig(calibration_year=2025, source_sample_fraction=0.1) assert config.sample_fraction == 1.0 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py index 8e1afae464..ebc332dc5a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_preparation.py @@ -14,7 +14,10 @@ from microcosm.graph import ContentStore, compile_graph, run_graph from test_support.microcosm_build.uk_atomic_support_fixtures import write_toy_supports from test_support.microcosm_build.uk_calibration_run import _bound_checkpoint -from test_support.microcosm_build.uk_full_build_cli import patch_support_register +from test_support.microcosm_build.uk_full_build_cli import ( + geography_first_sidecar_fields, + patch_support_register, +) from test_support.microcosm_build.uk_full_population_graph import source_frame from test_support.microcosm_build.uk_ladder_rowwise_clone import ( toy_ladder as toy_ladder, @@ -29,7 +32,9 @@ def checkpoint_argv(tmp_path, toy_ladder, monkeypatch): path = write_uk_national_frame(source_frame(), tmp_path / "spine.h5") frame, _ = load_uk_national_frame(path) sidecar_path, gates_path, sidecar = _bound_checkpoint(tmp_path, frame) - sidecar["stages"] = ["frs_spine"] + # A geography-first roster, which the dense line requires before it pins + # any input (microcosm#931). + sidecar.update(geography_first_sidecar_fields("frs_spine")) sidecar["sampling"] = {"fraction": 1.0, "seed": 7} sidecar_path.write_text(json.dumps(sidecar)) ledger = tmp_path / "ledger" @@ -135,7 +140,7 @@ def test_prepare_real_checkpoint_preserves_source_year_and_wires_preflight( "spine_provenance" ] provenance = json.loads(store.load_bytes(provenance_key)) - assert provenance["stages"] == ["frs_spine"] + assert provenance["stages"] == geography_first_sidecar_fields("frs_spine")["stages"] assert provenance["fit_weight_records"] == {"model": {"fit_weights_used": True}} fresh = cli.prepare_full_build(args) replay = run_graph( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py index bbb8be9a54..91bc1f2ced 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py @@ -43,6 +43,9 @@ def test_graph_driver_dry_run_prints_the_operation_inventory( from microcosm.build.uk_runtime.chronicle_feed import load_uk_chronicle_feed from microcosm.build.uk_runtime.national_frame import load_uk_national_frame from test_support.microcosm_build.uk_calibration_run import _bound_checkpoint + from test_support.microcosm_build.uk_full_build_cli import ( + geography_first_sidecar_fields, + ) input_h5 = tmp_path / "spine.h5" ladder_path = tmp_path / "ladder.npz" @@ -51,7 +54,8 @@ def test_graph_driver_dry_run_prints_the_operation_inventory( _write_ladder(ladder_path) frame, _ = load_uk_national_frame(input_h5) sidecar_path, gates_path, sidecar = _bound_checkpoint(tmp_path, frame) - sidecar["stages"] = ["frs_spine"] + # The dense line plans only on a geography-first spine (microcosm#931). + sidecar.update(geography_first_sidecar_fields("frs_spine")) sidecar["sampling"] = {"fraction": 1.0, "seed": 578} sidecar_path.write_text(json.dumps(sidecar)) monkeypatch.setattr(spine_build, "_rules_engine", lambda: object()) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_posture.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_posture.py new file mode 100644 index 0000000000..74d0d21c61 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_spine_posture.py @@ -0,0 +1,337 @@ +"""The dense line's spine posture (microcosm#931): the geography-first names +and the roster check that refuses a spine lacking any of them. + +Every roster here is invented, read off the packaged spec, or written by the +spine build's own sidecar writer over a synthetic frame; no sidecar of a built +spine is read. +""" + +import ast +import inspect +import json +from types import MappingProxyType, SimpleNamespace + +import pytest + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.uk_runtime import geography_first_roster +from microcosm.build.uk_runtime.geography_first_roster import ( + GEOGRAPHY_FIRST_OPERATIONS, + GEOGRAPHY_FIRST_STAGES, + SIDECAR_GEOGRAPHY_KEYS, +) +from microcosm.build.uk_runtime.spine_posture import ( + require_geography_first_spine, + roster_failures, + spec_roster, + stage_roster, +) +from test_support.microcosm_build.uk_full_build_cli import ( + geography_first_sidecar_fields, + synthetic_spine_geography, +) + +#: What a geography-first spine runs that a spine built from the packaged spec +#: does not yet: the area stages and the main-residence localiser. The +#: geography declarations shorten this list until it is empty. +_PACKAGED_ROSTER_LACKS = ( + "stage atomic_area_assign", + "stage local_authority", + "stage rfdr", + "stage area_brma", + "stage spi_copy_areas", + "stage private_rent", + "operation localise_main_residence_value in was_wealth", +) + + +def _passing(): + roster = geography_first_sidecar_fields("frs_spine") + return roster["stages"], roster["operations"], roster["geography"] + + +def test_the_leaf_names_the_stages_the_operation_and_the_sidecar_keys(): + assert GEOGRAPHY_FIRST_STAGES == ( + "geographic_expand", + "geographic_identity", + "atomic_area_assign", + "local_authority", + "rfdr", + "area_brma", + "spi_copy_areas", + "private_rent", + ) + assert isinstance(GEOGRAPHY_FIRST_OPERATIONS, MappingProxyType) + assert dict(GEOGRAPHY_FIRST_OPERATIONS) == { + "was_wealth": ("localise_main_residence_value",) + } + assert SIDECAR_GEOGRAPHY_KEYS == ( + "assignment", + "identity", + "definition_sha256", + "support_pins", + "stream", + "register", + ) + for names in ( + GEOGRAPHY_FIRST_STAGES, + SIDECAR_GEOGRAPHY_KEYS, + *GEOGRAPHY_FIRST_OPERATIONS.values(), + ): + assert isinstance(names, tuple) and len(set(names)) == len(names) + # The localiser runs inside a stage that is not one of the roster's own. + assert not set(GEOGRAPHY_FIRST_OPERATIONS) & set(GEOGRAPHY_FIRST_STAGES) + with pytest.raises(TypeError): + GEOGRAPHY_FIRST_OPERATIONS["was_lisa"] = () # type: ignore[index] + + +def test_the_leaf_imports_nothing_first_party(): + """A names-only leaf: a module that reads it reaches no stage code.""" + imported = set() + for node in ast.walk(ast.parse(inspect.getsource(geography_first_roster))): + if isinstance(node, ast.ImportFrom): + assert node.level == 0, node.module + imported.add(node.module) + elif isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + assert imported == {"__future__", "collections.abc", "types"} + + +def test_each_roster_stage_is_the_stage_its_transform_runs_as(): + """The leaf restates the names its stages' transforms declare, since it + imports none of them; this holds the two together.""" + from microcosm.build.uk_runtime import ( + geographic_expand, + geographic_identity, + spine_geography, + spine_private_rent, + ) + + assert GEOGRAPHY_FIRST_STAGES == ( + geographic_expand.GEOGRAPHIC_EXPAND_STAGE_NAME, + geographic_identity.GEOGRAPHIC_IDENTITY_STAGE_NAME, + spine_geography.ATOMIC_AREA_ASSIGN_STAGE_NAME, + spine_geography.LOCAL_AUTHORITY_STAGE_NAME, + spine_geography.RFDR_STAGE_NAME, + spine_geography.AREA_BRMA_STAGE_NAME, + spine_geography.SPI_COPY_AREAS_STAGE_NAME, + spine_private_rent.PRIVATE_RENT_STAGE_NAME, + ) + + +def test_the_packaged_roster_is_refused_naming_what_it_lacks(monkeypatch): + """A spine built on this code lacks the area stages and the localiser, + and the sidecar the spine build's own writer records for it has no + geography block: so the dense line refuses every spine until the + geography declarations land (microcosm#931). The stages the spine already + runs pass.""" + from microcosm.build.uk_runtime import spine_build + from test_support.microcosm_build.uk_full_population_graph import source_frame + + spec = load_country_spec("uk") + stages, operations = spec_roster(spec) + assert {"geographic_expand", "geographic_identity", "was_wealth"} <= set(stages) + assert roster_failures(stages, operations, from_sidecar=False) == ( + _PACKAGED_ROSTER_LACKS + ) + # The writer every spine build calls, over a synthetic frame and the + # stages in the spine graph's order, read back through JSON as the dense + # preparation reads it. Only the engine's provenance stands in. + monkeypatch.setattr( + spine_build, "_rules_engine_provenance", lambda: {"version": "stand-in"} + ) + by_name = spec.sources.stage_map() + written = spine_build._build_sidecar( + frame=source_frame(), + stages=[by_name[name] for name in spine_build._uk_spine_stage_names(spec)], + records=[], + artifact_pins={}, + resource_pins={}, + input_artifact_pins={}, + hmrc_replay={}, + stochastic_contract_sha256="0" * 64, + frs_vintage="stand-in", + sampling=None, + ) + sidecar = json.loads(json.dumps(written)) + recorded = (sidecar["stages"], sidecar["operations"], sidecar.get("geography")) + assert set(sidecar["stages"]) == set(stages) + assert roster_failures(*recorded) == (*_PACKAGED_ROSTER_LACKS, "geography block") + with pytest.raises(ValueError, match="builds only on a geography-first spine"): + require_geography_first_spine(*recorded) + + +def test_spec_roster_reads_the_declared_names_and_kinds_in_order(): + def operation(kind): + return SimpleNamespace(kind=kind) + + spec = SimpleNamespace( + sources=SimpleNamespace( + stages=( + SimpleNamespace( + stage="second", operations=(operation("b"), operation("a")) + ), + SimpleNamespace(stage="first", operations=()), + ) + ) + ) + assert spec_roster(spec) == ( + ("second", "first"), + {"second": ("b", "a"), "first": ()}, + ) + # The same reading of any declared stages, a composed spine's among them, + # in the order they come. + assert stage_roster(iter(spec.sources.stages)) == spec_roster(spec) + assert stage_roster(reversed(spec.sources.stages)) == ( + ("first", "second"), + {"first": (), "second": ("b", "a")}, + ) + with pytest.raises(ValueError, match="declares no source stages"): + spec_roster(SimpleNamespace(sources=None)) + + +def test_a_geography_first_roster_passes_and_extra_names_never_fail(): + stages, operations, geography = _passing() + assert roster_failures(stages, operations, geography) == () + assert require_geography_first_spine(stages, operations, geography) is None + # The stand-in's block holds the leaf's keys, each once. + assert tuple(synthetic_spine_geography()) == SIDECAR_GEOGRAPHY_KEYS + wider = ( + ["frs_take_up", *stages, "nts_bus_travel"], + { + **operations, + "was_wealth": ["derive", *operations["was_wealth"], "support_clip"], + "nts_bus_travel": ["support_clip"], + }, + {**geography, "seed": 0}, + ) + assert roster_failures(*wider) == () + # Tuples and read-only mappings read as lists and dicts do. + assert ( + roster_failures( + tuple(stages), + MappingProxyType({key: tuple(kinds) for key, kinds in operations.items()}), + MappingProxyType(geography), + ) + == () + ) + # Only the block's keys are read, never their values. + assert roster_failures(stages, operations, dict.fromkeys(geography)) == () + + +def test_each_missing_name_fails_alone_in_the_leafs_order(): + stages, operations, geography = _passing() + for stage in GEOGRAPHY_FIRST_STAGES: + kept = [name for name in stages if name != stage] + assert roster_failures(kept, operations, geography) == (f"stage {stage}",) + for stage, required in GEOGRAPHY_FIRST_OPERATIONS.items(): + for kind in required: + fewer = { + **operations, + stage: [name for name in operations[stage] if name != kind], + } + assert roster_failures(stages, fewer, geography) == ( + f"operation {kind} in {stage}", + ) + # A stage the roster records no kinds for holds none, and an + # operation counts only in a stage the roster ran. + unlisted = { + name: kinds for name, kinds in operations.items() if name != stage + } + assert roster_failures(stages, unlisted, geography) == ( + f"operation {kind} in {stage}", + ) + unrun = [name for name in stages if name != stage] + assert roster_failures(unrun, operations, geography) == ( + f"operation {kind} in {stage}", + ) + for key in SIDECAR_GEOGRAPHY_KEYS: + fewer = {name: value for name, value in geography.items() if name != key} + assert roster_failures(stages, operations, fewer) == (f"geography key {key}",) + for block in (None, [], "atomic", ["assignment"]): + assert roster_failures(stages, operations, block) == ("geography block",) + assert roster_failures([], {}, {}) == ( + *(f"stage {stage}" for stage in GEOGRAPHY_FIRST_STAGES), + *( + f"operation {kind} in {stage}" + for stage, required in GEOGRAPHY_FIRST_OPERATIONS.items() + for kind in required + ), + *(f"geography key {key}" for key in SIDECAR_GEOGRAPHY_KEYS), + ) + + +def test_an_unreadable_roster_fails_by_its_name(): + """A roster that is not names fails as a whole, in place of the names it + would have held; a stage whose kinds are not names holds none.""" + stages, operations, geography = _passing() + for bad in (None, "geographic_expand", b"geographic_expand", {"x": 1}, [1]): + assert roster_failures(bad, operations, geography) == ("stage roster",) + assert roster_failures([*stages, None], operations, geography) == ("stage roster",) + assert roster_failures(None, operations, None) == ( + "stage roster", + "geography block", + ) + for bad in (None, [], ["was_wealth"], "was_wealth"): + assert roster_failures(stages, bad, geography) == ("operation roster",) + for kinds in ( + None, + "localise_main_residence_value", + [None], + {"localise_main_residence_value": True}, + ): + assert roster_failures( + stages, {**operations, "was_wealth": kinds}, geography + ) == ("operation localise_main_residence_value in was_wealth",) + + +def test_a_roster_read_without_a_sidecar_reads_no_geography_block(): + """A spine request's spine runs inside the dense build's own graph, so + its roster has no block; one passed with it is a caller's error.""" + stages, operations, geography = _passing() + assert roster_failures(stages, operations, from_sidecar=False) == () + assert roster_failures([], {}, from_sidecar=False) == ( + tuple( + failure + for failure in roster_failures([], {}, {}) + if not failure.startswith("geography key ") + ) + ) + with pytest.raises(ValueError, match="no geography block to check"): + roster_failures(stages, operations, geography, from_sidecar=False) + with pytest.raises(ValueError, match="no geography block to check"): + require_geography_first_spine(stages, operations, geography, from_sidecar=False) + + +def test_the_refusal_names_what_the_roster_lacks_and_nothing_else(): + stages, operations, geography = _passing() + kept = [name for name in stages if name not in {"area_brma", "private_rent"}] + fewer = {key: value for key, value in geography.items() if key != "stream"} + with pytest.raises(ValueError) as sidecar: + require_geography_first_spine(kept, operations, fewer) + assert str(sidecar.value) == ( + "The dense line builds only on a geography-first spine, one that " + "assigns every geographic clone its area (microcosm#931); the bound " + "spine's build sidecar lacks: stage area_brma; stage private_rent; " + "geography key stream." + ) + with pytest.raises(ValueError) as request: + require_geography_first_spine(kept, operations, from_sidecar=False) + assert str(request.value).endswith( + "the spine request's stage roster lacks: stage area_brma; stage private_rent." + ) + # Names only: what else the roster holds, figures among it, never enters + # the refusal, which names the leaf's missing names alone. + with pytest.raises(ValueError) as figures: + require_geography_first_spine( + [*kept, "stage_with_4321_rows"], + {**operations, "stage_with_4321_rows": ["kind_9876"]}, + {**fewer, "support_pins": {"uk_system": {"size_bytes": 5555}}}, + ) + assert str(figures.value) == str(sidecar.value) + with pytest.raises(ValueError) as everything: + require_geography_first_spine([], {}, {}) + assert "geography key definition_sha256" in str(everything.value) + for refused in (sidecar, request, everything): + text = str(refused.value).replace("microcosm#931", "").replace("sha256", "") + assert not any(character.isdigit() for character in text) diff --git a/test_support/microcosm_build/uk_full_build_cli.py b/test_support/microcosm_build/uk_full_build_cli.py index 263b3ebebe..9874ed50fd 100644 --- a/test_support/microcosm_build/uk_full_build_cli.py +++ b/test_support/microcosm_build/uk_full_build_cli.py @@ -1,11 +1,12 @@ """Synthetic dense full-build fixtures for the canonical UK CLI: the -kernel stand-ins, payload builders, the prepared build and the -``graph_dense_bundle`` runner the release pre-flight and assembler tests -feed.""" +kernel stand-ins, payload builders, the prepared build, the stand-in spine +roster and the ``graph_dense_bundle`` runner the release pre-flight and +assembler tests feed.""" # ruff: noqa: F401 import base64 +import copy import hashlib import importlib.util import json @@ -34,6 +35,11 @@ classify_full_gate_outcomes, uk_full_gate_manifest, ) +from microcosm.build.uk_runtime.geography_first_roster import ( + GEOGRAPHY_FIRST_OPERATIONS, + GEOGRAPHY_FIRST_STAGES, + SIDECAR_GEOGRAPHY_KEYS, +) from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig, UKFullGraph from microcosm.build.uk_runtime.graph_calibration import ( SIZE_RECEIPT_TYPE, @@ -122,6 +128,51 @@ def patch_support_register(monkeypatch, pins=None) -> None: ) +#: A stand-in sidecar ``geography`` block's values, by key. The dense line's +#: posture check reads the block's keys only (microcosm#931), so the values are +#: synthetic stand-ins, not the spine build's. +_SYNTHETIC_SPINE_GEOGRAPHY = { + "assignment": "atomic", + "identity": "geography_household_key", + "definition_sha256": "f" * 64, + "support_pins": SYNTHETIC_SUPPORT_PINS, + "stream": ["sha256-u53-v1", "uk-spine-atomic-area-v1", 0, 0], + "register": "synthetic", +} + + +def synthetic_spine_geography() -> dict: + """A stand-in sidecar ``geography`` block under the leaf's keys, each once. + + A key the leaf gains without a stand-in value here fails loudly. + """ + return { + key: copy.deepcopy(_SYNTHETIC_SPINE_GEOGRAPHY[key]) + for key in SIDECAR_GEOGRAPHY_KEYS + } + + +def geography_first_sidecar_fields(*stages: str) -> dict: + """The roster fields of a stand-in build sidecar the dense line accepts. + + ``stages`` run first, then every geography-first stage and every stage a + geography-first operation runs in that ``stages`` does not name. Each stage + records the operation kinds the leaf requires of it and no other, and the + sidecar records a stand-in ``geography`` block (microcosm#931). Merge it + into a stand-in sidecar. + """ + names = list( + dict.fromkeys((*stages, *GEOGRAPHY_FIRST_STAGES, *GEOGRAPHY_FIRST_OPERATIONS)) + ) + return { + "stages": names, + "operations": { + name: list(GEOGRAPHY_FIRST_OPERATIONS.get(name, ())) for name in names + }, + "geography": synthetic_spine_geography(), + } + + STEM = "microcosm_uk_2024_25_local" #: The UK gate signing key's variable and a synthetic 32-byte key: a #: release-candidate request is refused without one, and the dense build signs From 51eee4992ab7f8a35d86a8cdca175291341063d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:20:59 +0000 Subject: [PATCH 070/108] Retire the dense line's property, BRMA, private-rent and fuel-relief nodes (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dense line no longer localises home values, council-tax bands, BRMAs, private rents or Rural Fuel Duty Relief flags of its own. Its five kernels go (uk.full.property.value@1, uk.full.property.band@1, uk.full.geography.brma@1, uk.full.geography.rfdr@1 and uk.full.private_rent@1), with their nodes, configuration fields, flags, resource pins, bindings and release-candidate refusals. On the atomic assignment the dense geography is now the identity, the shared support imports, assignment and derivation, the local-authority resolve, the shared gate and the UK distribution gate. The dense line keeps the spine's council-tax bands. Why. María's ruling of 10 Nov (decision 4) holds dense and national builds back until (c) re-bases them, and C-21 re-bases the dense line in three commits, refusal first. The previous commit refuses, before the build pins or reads any other input, every spine that is not geography-first, so no dense build runs until the spine declares its area stages and its home-value localiser. This commit retires the four localisation families, whose work moves into the spine: the BRMA, fuel-relief and private-rent transforms, added unwired in (c) commit 7, become the spine's area_brma, rfdr and private_rent stages with the geography declarations (commit 8), and England and Wales home values are localised inside the WAS chain (commit 11). No (c) commit re-draws the council-tax band. The next commit builds the dense line on the bound spine's clones, areas and sampling. - graph_population: the five kernels and the helpers that composed their nodes go, with the property quantile artifact type (microcosm.uk.property-quantile-draw), the re-exported mode tuples, the column lists, and the pool checkpoint's typed edge to the value node and its property_localisation receipt block. append_uk_population_nodes and _atomic_geography_nodes no longer take the four *_localisation_mode arguments or full_scale, and uk_pool_validation_inputs takes only the assignment. - graph_build: UKFullBuildConfig loses property_localisation, brma_localisation, private_rent_localisation and rfdr_localisation, their defaults and checks, and its refusal of brma target levels without area BRMAs. uk_full_graph no longer passes the modes, full_scale or brma_rows, so the dense target selection keeps Scotland's BRMA-grain rows on either assignment. They bind on the spine's brma column, which a spine the dense line accepts reads off its areas (a default, listed for María; the alternative leaves them out on the legacy assignment until the next commit). - graph_targets: append_uk_target_nodes' brma_rows switch and the target selection's brma_rows_left_out field go, with the reason text that named --brma-localisation none; nothing could set the switch once the configuration field went. - full_build_cli: sixteen flags retire. They are the four modes, --property-localisation, --brma-localisation, --private-rent-localisation and --rfdr-localisation; the resource overrides, --price-paid-quantiles, --voa-band-mix, --scotgov-band-mix, --brma-area-layer, --brma-rent-quantiles and --rfdr-area-layer, which nothing reads once the nodes go (a default, listed for María); and their pins, --price-paid-quantiles-sha256, --voa-band-mix-sha256, --scotgov-band-mix-sha256, --brma-area-layer-sha256, --brma-rent-quantiles-sha256 and --rfdr-area-layer-sha256. The parser refuses a script that passes one as an unrecognized argument. The four _prepare_*_localisation helpers and their resource tables go, so the seven resources leave the input pins and the run's sources, and the build request's bindings lose their four localisation blocks. The --target-geographies help says the brma rows bind on the BRMAs the spine reads off its areas. - rowwise_cli: the release-candidate refusals of the four none modes go. A comment says spine_posture holds every dense build, release candidates included, to the spine stages that set these columns before the build pins or reads any other input. - size_checkpoint: the four identity blocks that bound the resources' pins go. - orrery_contract: the five node ids leave the geography group. - brma_area_layer, rfdr_area_layer and private_rent_localisation: the unread BRMA_LOCALISATION_MODES, RFDR_LOCALISATION_MODES and PRIVATE_RENT_LOCALISATION_MODES go with their __all__ entries. private_rent_localisation's docstring names the spine's private-rent stage (spine_private_rent.UKPrivateRentStageTransform, G7) as the stage that ranks the renters and supplies the row keys and tie-break uniforms, and RFDR_COLUMN's comment names the spine's fuel-relief stage. No kernel or gate phase code hashes these three modules at this commit, so no key moves; once the geography declarations bind the transforms that import them, an edit here would re-key those stages and every spine node after them. - tools/uk_private_rent_pool.py and tools/uk_private_rent_receipt.py say they read only dense runs built before this change. Kept: every function the nodes called, with its tests (owner_uniforms, localise_values, localise_bands and spine_household_keys in property_localisation, area_brmas in brma_area_layer, area_flags in rfdr_area_layer and localise_rents in private_rent_localisation); the committed resources; and the terminal uk_local_brma_enum_domain gate, which the WAS-chain commit retires. The spine's geography and private-rent transforms, which no stage declares yet, call area_brmas, area_flags and localise_rents, and the regional_home_values stage calls owner_uniforms. No dataset build reads the VOA or Scottish band mix now. Left for later (stale text in files this commit does not edit, by rule or because spine stages or gate kernels hash them): - docs/uk-full-build-graph.md's flag paragraphs and node rows: (f); another commit of this PR is editing the file. - PROPERTY_LOCALISATION_MODES and the docstrings of property_localisation, high_value_home_support and regional_home_values, which spine stages hash; the rank-basis comments in graph.py, national_frame and graph_kernels; terminal_gates' fuel-relief allow-list comment; and battery_bindings' price-paid support-bound comment: commit 11, which localises home values inside the WAS chain and retires regional_home_values. - household_lineage's docstring, which also names the dense identity node: commit 4, which retires that node and re-keys every spine node anyway. - Declaration and generated texts, which this commit may not edit: gates.json's uk_local_brma_enum_domain notes, the high_value_home_support and regional_home_values notes in sources.yaml and source_stages.json, and the price-paid support bounds' policy go to commit 11, which retires that gate and regional_home_values and moves the value map into the WAS chain; the descriptions of the BRMA and fuel-relief layers (C-35), the BRMA rent quantiles and the VOA band mix, which their build tools write, and the Scottish band mix's purpose in its vendor selection and resource go to (f), as C-35 leaves the layers'. - regional_home_values still writes main_residence_value_rank_basis, which no node reads now; the release boundary drops it, and the WAS-chain commit retires the stage. What moves. Keys only; values: none. The dense line refuses every spine and the national line every clone-first one, no spine module changes, so K = 1 spine values cannot move, and a national build on a spine without clone indices computes what it did before. Of the 96 kernels the UK registries now register (101 before), 30 change hash, all uk.full: - the nine graph_population defines: uk.full.sample@1, uk.full.normalize@1, uk.full.expand@1, uk.full.identity@1, uk.full.locations@1, uk.full.geography_mapping@1, uk.full.geography.local_authority@1, uk.full.pool@1 and uk.full.geography_gate@1; - the fifteen that hash it: uk.full.measures@1, uk.full.problem@1, uk.full.dense@1, uk.full.calibrated@1, the five size kernels (uk.full.size_checkpoint_import@1, uk.full.size_search@1, uk.full.size_draw@1, uk.full.size_refit@1 and uk.full.selected@1), uk.full.rotated-holdout@1, uk.full-export.prepare@1, uk.full-export.prepare-inspection@1, uk.full.national_problem@1, uk.full.national-gates@1 and uk.full.national-readback@1; - uk.full.bound_spine@1, which hashes graph_build, where UKFullBuildConfig lives; - the five that hash graph_targets but not graph_population: uk.full.target_compilation@1, uk.full.target_selection@1, uk.full.national_targets@1, uk.full-gates.preflight@1 and uk.full-gates@1. The other 66 keep theirs: the 58 the spine registry holds (the create, claim and identity kernels and every stage), uk.spine-gates@1, the four shared atomic-geography kernels, the export package and read-back, and the certification readiness. Every kernel that hashes size_checkpoint also hashes graph_population; no kernel hashes full_build_cli, rowwise_cli, orrery_contract, brma_area_layer, rfdr_area_layer or private_rent_localisation. Of the gate phase codes only the full battery's terminal code moves, since it lists graph_population. Node keys, on composed graphs with one stand-in content key per source: - The atomic dense graph on a composed spine: the five retired nodes are gone, all 84 spine stage, boundary and gate nodes keep their keys, and all 23 dense nodes re-key: uk.full.sample, uk.full.normalize, uk.full.expand, uk.full.expand.owned, uk.full.identity, uk.full.geography.support.0, .1 and .2, uk.full.geography.assign, uk.full.geography.derive, uk.full.geography.gate, uk.full.geography.local_authority, uk.full.geography_gate, uk.full.target_compilation, uk.full.target_selection, uk.full.measures, uk.full.problem, uk.full.pool, uk.full.gates.preflight, uk.full.dense, uk.full.calibrated, uk.full.gates.calibrated and uk.full.holdout. The expand claim, the three support imports, assign, derive and the shared gate re-key through their inputs only; uk.full.gates.calibrated also through its terminal_code parameter. - The legacy dense graph on a composed spine: the 84 spine nodes keep their keys and all 17 dense nodes re-key, the same nodes with uk.full.locations and uk.full.geography_mapping in place of the identity and the atomic geography; uk.full.target_selection also loses its brma_rows parameter there. - The national graph on a stand-in bound spine: all six nodes re-key (uk.full.spine_checkpoint, uk.full.national_targets, uk.full.national_problem, uk.full.dense, uk.full.calibrated and uk.full.gates.calibrated), so a national build on an already built spine re-solves rather than reusing its store, with the same values. A dense run's input-pins digest, candidate identity digest and size-checkpoint identity no longer bind the seven resources. The 31 transport kernels that register without an engine keep their hashes. No declaration, generated file, gates.json or schema changes, and H2 is not regenerated. Tests (invented data): - Removed: test_uk_brma_area_graph.py and test_uk_rfdr_area_graph.py, which held node and configuration tests only; the graph sections of test_uk_private_rent_localisation.py (with its size-identity test), test_uk_property_localisation.py and test_uk_regional_home_values.py; and the CLI's five localisation tests. The function tests stay. - New: test_the_dense_localisation_flags_are_gone, one case per retired flag, each refused as an unrecognized argument; and test_the_dense_line_composes_and_rosters_no_localisation_node: the atomic dense graph on a composed spine composes none of the five nodes, neither declares nor reads their seven resources, and the Orrery groups place none of them. - Kept green: the stand-in source maps of test_support/microcosm_build/uk_full_target_graph.py and the engine-tier test_uk_full_graph_admission.py stop passing the retired resources; uk_full_population_graph.py's graph_and_registry drops the modes and full_scale; test_uk_rowwise_candidate's dry run drops two retired flags; and the toy writers only the nodes read leave uk_atomic_support_fixtures.py. test_country_spec's two comments on the Scottish band mix say no build reads it now. Checks: 54 engine-free files around the change: the 17 that are touched or test an edited module (778 passed, 2 skipped: unmounted supports), test_uk_graph.py in full (59 passed) and 36 adjacent files (1,320 passed, 9 skipped: unmounted supports and artifacts, absent Chronicle feeds), and test_country_spec.py once more after its last comment edit (144 passed). Through the engine slot, test_uk_full_target_graph, test_uk_full_graph_admission, test_uk_brma_area_layer and test_uk_country_adapter (13 passed). ruff check and ruff format --check on the 23 edited Python files, git diff --check, and ci_test_plan.py verify. test_uk_orrery_reconstruction, which fails on this machine only, was not run. An engine-free probe that reads code only compared this tree with its parent: every UK kernel's implementation hash, the edited modules each kernel and gate phase code hashes, and the node keys and parameters of three composed graphs (the atomic and legacy dense graphs on a composed spine with both gate batteries, and the national graph on a stand-in bound spine), with one stand-in content key per source. Co-Authored-By: Claude Opus 5.5 --- ...ense-localisation-nodes-retired.removed.md | 8 + .../build/uk_runtime/brma_area_layer.py | 4 - .../build/uk_runtime/full_build_cli.py | 347 +------ .../microcosm/build/uk_runtime/graph_build.py | 108 +-- .../build/uk_runtime/graph_population.py | 885 +----------------- .../build/uk_runtime/graph_targets.py | 32 +- .../build/uk_runtime/orrery_contract.py | 5 - .../uk_runtime/private_rent_localisation.py | 33 +- .../build/uk_runtime/rfdr_area_layer.py | 6 +- .../microcosm/build/uk_runtime/rowwise_cli.py | 16 +- .../build/uk_runtime/size_checkpoint.py | 40 - .../engine/uk/test_uk_full_graph_admission.py | 8 - .../engine_free/shared/test_country_spec.py | 6 +- .../engine_free/uk/test_uk_brma_area_graph.py | 161 ---- .../engine_free/uk/test_uk_full_build_cli.py | 175 +--- .../engine_free/uk/test_uk_orrery_contract.py | 69 ++ .../uk/test_uk_private_rent_localisation.py | 255 +---- .../uk/test_uk_property_localisation.py | 89 -- .../uk/test_uk_regional_home_values.py | 132 +-- .../engine_free/uk/test_uk_rfdr_area_graph.py | 191 ---- .../uk/test_uk_rowwise_candidate.py | 9 - .../uk_atomic_support_fixtures.py | 131 --- .../uk_full_population_graph.py | 22 +- .../microcosm_build/uk_full_target_graph.py | 14 +- tools/uk_private_rent_pool.py | 4 + tools/uk_private_rent_receipt.py | 9 +- 26 files changed, 161 insertions(+), 2598 deletions(-) create mode 100644 changelog.d/931-dense-localisation-nodes-retired.removed.md delete mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_brma_area_graph.py delete mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_rfdr_area_graph.py diff --git a/changelog.d/931-dense-localisation-nodes-retired.removed.md b/changelog.d/931-dense-localisation-nodes-retired.removed.md new file mode 100644 index 0000000000..86e6b3e6d4 --- /dev/null +++ b/changelog.d/931-dense-localisation-nodes-retired.removed.md @@ -0,0 +1,8 @@ +Retire the dense line's own property, BRMA, private-rent and fuel-relief nodes, with their flags, configuration fields and release-candidate refusals (#931). The dense line takes home values, BRMAs, private rents and Rural Fuel Duty Relief flags from the spine, whose own stages set them in later commits of #931 group (c), and keeps the spine's council-tax bands. It already refuses, before it pins or reads any other input, a spine whose roster lacks those stages (`spine_posture`), so no dense build runs until they land. + +- **Nodes and kernels.** `uk.full.property.value@1`, `uk.full.property.band@1`, `uk.full.geography.brma@1`, `uk.full.geography.rfdr@1` and `uk.full.private_rent@1` leave `microcosm.build.uk_runtime.graph_population`, with the helpers that composed them, the property quantile artifact type (`microcosm.uk.property-quantile-draw`) and the pool checkpoint's typed edge to it. `append_uk_population_nodes` no longer takes `property_localisation_mode`, `brma_localisation_mode`, `private_rent_localisation_mode`, `rfdr_localisation_mode` or `full_scale`, and `uk_pool_validation_inputs` no longer takes a property mode. On the atomic assignment the dense graph's geography is now the identity, the shared support imports, assignment and derivation, the local-authority resolve, the shared gate and the UK distribution gate. +- **Configuration.** `UKFullBuildConfig` loses `property_localisation`, `brma_localisation`, `private_rent_localisation` and `rfdr_localisation`, and with them its refusal of BRMA target rows without area BRMAs. The dense line no longer leaves its BRMA-grain rows out on either assignment: they bind on the spine's `brma` column, which a spine the dense line accepts reads off its areas. So `append_uk_target_nodes` loses its `brma_rows` switch and the target selection's payload its `brma_rows_left_out` field, which nothing could set once the configuration field went. +- **Flags.** `microcosm-build-uk` (`tools/build_uk_full.py`) no longer takes `--property-localisation`, `--price-paid-quantiles`, `--voa-band-mix`, `--scotgov-band-mix`, `--brma-localisation`, `--brma-area-layer`, `--private-rent-localisation`, `--brma-rent-quantiles`, `--rfdr-localisation` or `--rfdr-area-layer`, nor the six resource pins `--price-paid-quantiles-sha256`, `--voa-band-mix-sha256`, `--scotgov-band-mix-sha256`, `--brma-area-layer-sha256`, `--brma-rent-quantiles-sha256` and `--rfdr-area-layer-sha256`. A script that still passes one is refused as an unrecognized argument. The release-candidate refusals of the four `none` modes go with them: the spine-roster check holds every dense build, release candidates included, to a spine with the stages that set those columns. The build request's bindings lose their four localisation blocks, and the size-checkpoint identity loses the four blocks that bound those resources' pins. +- **Kept.** Every function the nodes called stays, with its tests: `owner_uniforms`, `localise_values`, `localise_bands` and `spine_household_keys` in `property_localisation`, `area_brmas` in `brma_area_layer`, `area_flags` in `rfdr_area_layer` and `localise_rents` in `private_rent_localisation`. The spine's geography and private-rent transforms (`spine_geography`, `spine_private_rent`), which no stage declares yet, call `area_brmas`, `area_flags` and `localise_rents`, and the `regional_home_values` stage calls `owner_uniforms`. The committed resources the nodes read and the terminal `uk_local_brma_enum_domain` gate stay too, though no dataset build reads the VOA or Scottish band mix now. The unread mode tuples `BRMA_LOCALISATION_MODES`, `RFDR_LOCALISATION_MODES` and `PRIVATE_RENT_LOCALISATION_MODES` go; `PROPERTY_LOCALISATION_MODES` stays for now, since spine stages hash `property_localisation`. `private_rent_localisation`'s docstring now names the spine's private-rent stage as the caller that supplies the row keys and tie-break uniforms. `tools/uk_private_rent_pool.py` and `tools/uk_private_rent_receipt.py` now say they read only dense runs built before this change. +- **What moves.** Keys only, no value: the dense line already refuses every spine, the national line every clone-first one, no spine module changes, and a national build on a spine without clone indices computes what it did before. Of the 96 kernels the UK registries register (101 before), 30 change hash, all `uk.full`: the nine `graph_population` defines (sample, normalisation, expand, identity, locations, mapping, local authority, pool and the distribution gate); the fifteen that hash it (the dense measures and problem, the dense solve, the calibration install and the five size kernels, the rotated holdout, both export preparations, and the national problem, gates and read-back); the bound spine's checkpoint kernel (`uk.full.bound_spine@1`), which hashes `graph_build`; and the five that hash `graph_targets` but not `graph_population` (the target compilation and selection, the national targets, and the full battery's preflight and calibrated gate kernels). Each kernel that hashes `size_checkpoint` also hashes `graph_population`, and no kernel hashes `full_build_cli`, `rowwise_cli`, `orrery_contract`, `brma_area_layer`, `rfdr_area_layer` or `private_rent_localisation`. Of the gate phase codes only the full battery's terminal code moves, since it lists `graph_population`, so the calibrated gates' `terminal_code` parameter changes too. On a spine the dense build composes, every spine stage and both spine gates keep their keys and every dense node re-keys; on a bound spine the checkpoint re-keys, and every dense and national node after it. A dense run's input pins no longer carry the seven resources, so its input-pins digest, candidate identity digest and size-checkpoint identity change. No declaration, generated file, `gates.json` or schema changes. +- **Tests.** The node and configuration tests go: `test_uk_brma_area_graph.py` and `test_uk_rfdr_area_graph.py` are removed, as are the graph sections of `test_uk_private_rent_localisation.py`, `test_uk_property_localisation.py` and `test_uk_regional_home_values.py` and the CLI's localisation flag tests. Their function tests stay. New tests check that each retired flag is refused as an unrecognized argument, and that the atomic dense graph composes none of the five nodes, neither declares nor reads their resources, and the Orrery roster names none of them. The toy resource writers that only the nodes read leave `test_support/microcosm_build/uk_atomic_support_fixtures.py`. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py index eff34a6b3a..f458316f49 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py @@ -34,7 +34,6 @@ __all__ = [ "BRMA_AREA_LAYER_SOURCE", - "BRMA_LOCALISATION_MODES", "GRAINS", "KIND", "METHOD_VERSION", @@ -54,9 +53,6 @@ METHOD_VERSION = "uk-brma-area-layer-v1" #: Graph source name of the committed layer bytes. BRMA_AREA_LAYER_SOURCE = "uk_brma_area_layer" -#: ``area`` reads each placed Great Britain household's BRMA off its assigned -#: area; ``none`` leaves the spine's regional draw on every clone. -BRMA_LOCALISATION_MODES = ("none", "area") NI_SYSTEM = "uk_ni_data_zone_2021" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 4fbbb8655f..30b3fc5ad3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -47,7 +47,6 @@ from collections.abc import Callable, Collection, Mapping from dataclasses import asdict, dataclass, replace from datetime import UTC, date, datetime -from importlib.resources import files from pathlib import Path from typing import TYPE_CHECKING, Protocol @@ -114,8 +113,6 @@ from .atomic_area_support import ( SYSTEMS as ATOMIC_SUPPORT_SYSTEMS, ) -from .brma_area_layer import BRMA_AREA_LAYER_SOURCE -from .brma_area_layer import METHOD_VERSION as BRMA_AREA_LAYER_METHOD from .calibration_run import ( _write_json, load_bound_spine_checkpoint, @@ -161,13 +158,7 @@ solved_national_registry, uk_national_graph, ) -from .graph_population import ( - BRMA_LOCALISATION_MODES, - GEOGRAPHY_ASSIGNMENTS, - PRIVATE_RENT_LOCALISATION_MODES, - PROPERTY_LOCALISATION_MODES, - RFDR_LOCALISATION_MODES, -) +from .graph_population import GEOGRAPHY_ASSIGNMENTS from .graph_targets import TARGET_SELECTION_TYPE, registry_from_payload from .graph_terminal import ( FULL_DIAGNOSTICS_CSV_TYPE, @@ -199,21 +190,6 @@ ) from .national_sampling import UK_SAMPLE_RUNG_TOKENS from .orrery_contract import persist_uk_run_evidence, save_uk_graph_schema -from .private_rent_localisation import COUNTS_SOURCE as PRIVATE_RENTER_COUNTS_SOURCE -from .private_rent_localisation import METHOD_VERSION as PRIVATE_RENT_METHOD -from .private_rent_localisation import QUANTILES_SOURCE as RENT_QUANTILES_SOURCE -from .property_localisation import ( - BAND_MIX_SOURCE, - QUANTILES_SOURCE, - SCOTLAND_BAND_MIX_RESOURCE, - SCOTLAND_BAND_MIX_SOURCE, - SUPPORT_STRATUM_LOWER, -) -from .property_localisation import ( - METHOD_VERSION as PROPERTY_LOCALISATION_METHOD, -) -from .rfdr_area_layer import METHOD_VERSION as RFDR_AREA_LAYER_METHOD -from .rfdr_area_layer import RFDR_AREA_LAYER_SOURCE from .rowwise_cli import ( MANIFEST_FILENAME, REPOSITORY, @@ -511,98 +487,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=sha256_argument, dest=f"atomic_support_sha256_{label}", ) - parser.add_argument( - "--property-localisation", - choices=PROPERTY_LOCALISATION_MODES, - default="price_paid", - help=( - "Dense role: price_paid (default) reads each owner clone's home value " - "off its assigned constituency's uprated Land Registry sale distribution " - "and its council-tax band off its LSOA's VOA mix (microcosm#1240), or a " - "Scottish owner clone's off its council area's CTAXBASE mix " - "(microcosm#1095 B12); none " - "keeps the spine's values and bands on every clone, for measurement " - "builds only." - ), - ) - for name, label in _PROPERTY_RESOURCE_ARGUMENTS.items(): - parser.add_argument( - f"--{label}", - type=Path, - dest=label.replace("-", "_"), - default=None, - help=( - f"Resource bytes for the graph source {name!r} (default: the " - "committed UK package resource)." - ), - ) - parser.add_argument( - f"--{label}-sha256", - type=sha256_argument, - dest=f"{label.replace('-', '_')}_sha256", - ) - parser.add_argument( - "--brma-localisation", - choices=BRMA_LOCALISATION_MODES, - default="area", - help=( - "Dense role: area (default) reads each placed Great Britain clone's BRMA " - "off its assigned LSOA or output area (microcosm#1262); none keeps the " - "spine's regional draw on every clone, for measurement builds only." - ), - ) - parser.add_argument( - "--brma-area-layer", - type=Path, - default=None, - help=( - "Resource bytes for the graph source 'uk_brma_area_layer' (default: the " - "committed UK package resource)." - ), - ) - parser.add_argument("--brma-area-layer-sha256", type=sha256_argument) - parser.add_argument( - "--rfdr-localisation", - choices=RFDR_LOCALISATION_MODES, - default=None, - help=( - "Dense role: area (the default with the atomic assignment) sets each " - "placed household's Rural Fuel Duty Relief flag off its assigned output " - "area (microcosm#1418); none (the default with the legacy assignment) " - "leaves the engine's default, for measurement builds only." - ), - ) - parser.add_argument( - "--rfdr-area-layer", - type=Path, - default=None, - help=( - "Resource bytes for the graph source 'uk_rfdr_area_layer' (default: the " - "committed UK package resource)." - ), - ) - parser.add_argument("--rfdr-area-layer-sha256", type=sha256_argument) - parser.add_argument( - "--private-rent-localisation", - choices=PRIVATE_RENT_LOCALISATION_MODES, - default=None, - help=( - "Dense role: brma_list (the default with area BRMAs) re-draws each Great " - "Britain private renter's rent from its assigned BRMA's list of rents at " - "its FRS rank and region level (microcosm#1262); none (the default " - "without them) keeps the spine's rents, for measurement builds only." - ), - ) - parser.add_argument( - "--brma-rent-quantiles", - type=Path, - default=None, - help=( - "Resource bytes for the graph source 'uk_brma_rent_quantiles' (default: " - "the committed UK package resource)." - ), - ) - parser.add_argument("--brma-rent-quantiles-sha256", type=sha256_argument) parser.add_argument( "--ledger-facts", type=Path, @@ -644,7 +528,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=_target_geographies, default=None, metavar="all|country,...", - help="Default all: calibrate all applicable geographies together. country is an explicit filter in this same build; brma (Scotland's BRMA rent rows) needs area BRMAs.", + help="Default all: calibrate all applicable geographies together. country is an explicit filter in this same build; brma (Scotland's BRMA rent rows) binds on the BRMAs the spine reads off its areas.", ) parser.add_argument( "--households-only", @@ -977,221 +861,6 @@ def _input_record(path: Path, pin: dict, *, pinned: bool) -> dict: } -#: Graph source name to command-line label of the property-localisation -#: resources (microcosm#1240; the Scottish band mix, microcosm#1095 B12). Each -#: defaults to the committed package file. -_PROPERTY_RESOURCE_ARGUMENTS = { - QUANTILES_SOURCE: "price-paid-quantiles", - BAND_MIX_SOURCE: "voa-band-mix", - SCOTLAND_BAND_MIX_SOURCE: "scotgov-band-mix", -} -_PROPERTY_RESOURCE_FILES = { - QUANTILES_SOURCE: "high_value_home_value_knots.json", - BAND_MIX_SOURCE: "voa_council_tax_band_mix_lsoa.json", - SCOTLAND_BAND_MIX_SOURCE: SCOTLAND_BAND_MIX_RESOURCE, -} - - -def _packaged_property_resource(name: str) -> Path: - return Path( - str(files("microcosm.build.uk").joinpath(_PROPERTY_RESOURCE_FILES[name])) - ) - - -def _prepare_property_localisation( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> dict: - """Pin the knots and band-mix resources and record the localisation binding. - - Each resource enters ``pins`` under its graph source name, so the Logbook - input-pins digest, the candidate identity digest and the size-checkpoint - identity all bind it. A release candidate must build on the committed - package resources, whatever path the operator named: the committed bytes - are the reviewed ones. ``none`` binds nothing here. - """ - mode = getattr(args, "property_localisation", "none") - if mode == "none": - return {"mode": "none"} - resource_pins = {} - for name, label in _PROPERTY_RESOURCE_ARGUMENTS.items(): - attribute = label.replace("-", "_") - path = getattr(args, attribute, None) or _packaged_property_resource(name) - pin = _pin(path, getattr(args, f"{attribute}_sha256", None)) - if getattr(args, "release_candidate", False): - committed = _pin(_packaged_property_resource(name)) - if pin != committed: - raise ValueError( - f"--release-candidate requires the committed {name!r} resource: " - f"sha256 {pin['sha256']} is not the package's {committed['sha256']}." - ) - pins[name] = pin - resource_pins[name] = pin - sources[name] = path - return { - "mode": mode, - "method": PROPERTY_LOCALISATION_METHOD, - "support_stratum_lower": SUPPORT_STRATUM_LOWER, - "resource_pins": resource_pins, - } - - -def _prepare_brma_localisation( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> dict: - """Pin the area-to-BRMA layer and record the BRMA binding (microcosm#1262). - - The layer enters ``pins`` under its graph source name, as the property - resources do, and a release candidate must build on the committed bytes. - ``none`` binds nothing here. - """ - mode = getattr(args, "brma_localisation", "none") - if mode == "none": - if getattr(args, "brma_area_layer", None) or getattr( - args, "brma_area_layer_sha256", None - ): - raise ValueError( - "--brma-area-layer and --brma-area-layer-sha256 need " - "--brma-localisation area: without it nothing reads the layer." - ) - return {"mode": "none"} - committed_path = Path( - str(files("microcosm.build.uk").joinpath("brma_area_layer.json")) - ) - path = getattr(args, "brma_area_layer", None) or committed_path - pin = _pin(path, getattr(args, "brma_area_layer_sha256", None)) - if getattr(args, "release_candidate", False): - committed = _pin(committed_path) - if pin != committed: - raise ValueError( - "--release-candidate requires the committed BRMA area layer: " - f"sha256 {pin['sha256']} is not the package's {committed['sha256']}." - ) - pins[BRMA_AREA_LAYER_SOURCE] = pin - sources[BRMA_AREA_LAYER_SOURCE] = path - return { - "mode": mode, - "method": BRMA_AREA_LAYER_METHOD, - "resource_pins": {BRMA_AREA_LAYER_SOURCE: pin}, - } - - -def _prepare_rfdr_localisation( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> dict: - """Pin the Rural Fuel Duty Relief area layer and record its binding. - - As with the BRMA layer, the layer enters ``pins`` under its graph source - name and a release candidate must build on the committed bytes - (microcosm#1418). ``none`` binds nothing here. - """ - mode = getattr(args, "rfdr_localisation", None) - if mode is None: - # Unset, the mode follows the geography assignment, as the build config - # resolves it (the independent review of microcosm#1420). - mode = ( - "area" - if getattr(args, "geography_assignment", "atomic") == "atomic" - else "none" - ) - args.rfdr_localisation = mode - if mode == "none": - if getattr(args, "rfdr_area_layer", None) or getattr( - args, "rfdr_area_layer_sha256", None - ): - raise ValueError( - "--rfdr-area-layer and --rfdr-area-layer-sha256 need " - "--rfdr-localisation area: without it nothing reads the layer." - ) - return {"mode": "none"} - committed_path = Path( - str(files("microcosm.build.uk").joinpath("rfdr_area_layer.json")) - ) - path = getattr(args, "rfdr_area_layer", None) or committed_path - pin = _pin(path, getattr(args, "rfdr_area_layer_sha256", None)) - if getattr(args, "release_candidate", False): - committed = _pin(committed_path) - if pin != committed: - raise ValueError( - "--release-candidate requires the committed RFDR area layer: " - f"sha256 {pin['sha256']} is not the package's {committed['sha256']}." - ) - pins[RFDR_AREA_LAYER_SOURCE] = pin - sources[RFDR_AREA_LAYER_SOURCE] = path - return { - "mode": mode, - "method": RFDR_AREA_LAYER_METHOD, - "resource_pins": {RFDR_AREA_LAYER_SOURCE: pin}, - } - - -#: Graph source name to committed package file of the private-rent node's -#: resources (microcosm#1262). Only the quantiles take an override. -_PRIVATE_RENT_RESOURCE_FILES = { - RENT_QUANTILES_SOURCE: "brma_rent_quantiles.json", - PRIVATE_RENTER_COUNTS_SOURCE: "brma_private_rented_households.json", -} - - -def _prepare_private_rent_localisation( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> dict: - """Pin the rent quantiles and census counts and record the rent binding. - - As with the property resources, each enters ``pins`` under its graph source - name and a release candidate must build on the committed bytes. ``none`` - binds nothing here. - """ - mode = getattr(args, "private_rent_localisation", None) - if mode is None: - # Unset, the mode follows the BRMA mode, as the build config resolves it. - mode = ( - "brma_list" - if getattr(args, "brma_localisation", "none") == "area" - else "none" - ) - args.private_rent_localisation = mode - if mode == "none": - if getattr(args, "brma_rent_quantiles", None) or getattr( - args, "brma_rent_quantiles_sha256", None - ): - raise ValueError( - "--brma-rent-quantiles and --brma-rent-quantiles-sha256 need " - "--private-rent-localisation brma_list: without it nothing " - "reads the quantiles." - ) - return {"mode": "none"} - resource_pins = {} - for name, filename in _PRIVATE_RENT_RESOURCE_FILES.items(): - committed_path = Path(str(files("microcosm.build.uk").joinpath(filename))) - override = ( - getattr(args, "brma_rent_quantiles", None) - if name == RENT_QUANTILES_SOURCE - else None - ) - expected = ( - getattr(args, "brma_rent_quantiles_sha256", None) - if name == RENT_QUANTILES_SOURCE - else None - ) - path = override or committed_path - pin = _pin(path, expected) - if getattr(args, "release_candidate", False): - committed = _pin(committed_path) - if pin != committed: - raise ValueError( - f"--release-candidate requires the committed {name!r} resource: " - f"sha256 {pin['sha256']} is not the package's {committed['sha256']}." - ) - pins[name] = pin - resource_pins[name] = pin - sources[name] = path - return { - "mode": mode, - "method": PRIVATE_RENT_METHOD, - "resource_pins": resource_pins, - } - - def _prepare_geography( args: argparse.Namespace, pins: dict, sources: dict[str, Path] ) -> tuple[dict | None, dict]: @@ -1476,10 +1145,6 @@ def prepare_full_build( # The atomic-area supports are admitted after the spine so a spine # refusal (sidecar, digest) is reported before a support one. definition, geography_binding = _prepare_geography(args, pins, sources) - property_binding = _prepare_property_localisation(args, pins, sources) - brma_binding = _prepare_brma_localisation(args, pins, sources) - private_rent_binding = _prepare_private_rent_localisation(args, pins, sources) - rfdr_binding = _prepare_rfdr_localisation(args, pins, sources) stage( telemetry, "input_pinning", @@ -1507,10 +1172,6 @@ def prepare_full_build( source_lineage_modulus=args.source_lineage_modulus, geography_assignment=args.geography_assignment, source_vintage=release.vintage, - property_localisation=args.property_localisation, - brma_localisation=args.brma_localisation, - private_rent_localisation=args.private_rent_localisation, - rfdr_localisation=args.rfdr_localisation, calibration=UKGraphCalibrationConfig( epochs=args.epochs, learning_rate=args.learning_rate, @@ -1642,10 +1303,6 @@ def prepare_full_build( "paired_ladder_sha256": pins["ladder"]["sha256"], }, "geography": geography_binding, - "property_localisation": property_binding, - "brma_localisation": brma_binding, - "private_rent_localisation": private_rent_binding, - "rfdr_localisation": rfdr_binding, } graph = add_uk_export_preparation( graph, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py index 77a483554e..2b376ab810 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -86,31 +86,6 @@ class UKFullBuildConfig: geography_assignment: str = "atomic" #: FRS release vintage folded into every geography identity key. source_vintage: str = field(default_factory=lambda: load_uk_frs_release().vintage) - #: ``price_paid`` reads each owner clone's home value off its assigned - #: constituency's uprated Land Registry sale distribution and its band off - #: its LSOA's VOA mix (microcosm#1240); ``none`` keeps the spine's values - #: and bands on every clone (measurement builds only; a release candidate - #: refuses it). Unset, the mode follows the geography assignment: - #: ``price_paid`` on the atomic assignment, ``none`` on the legacy one. - property_localisation: str | None = None - #: ``area`` reads each placed Great Britain household's BRMA off its - #: assigned area (microcosm#1262); ``none`` keeps the spine's regional - #: draw on every clone (measurement builds only; a release candidate - #: refuses it). Unset, the mode follows the geography assignment: ``area`` - #: on the atomic assignment, ``none`` on the legacy one. - brma_localisation: str | None = None - #: ``brma_list`` re-draws each Great Britain private renter's rent from its - #: assigned BRMA's list of rents at its FRS rank and region level - #: (microcosm#1262); ``none`` keeps the spine's rents (measurement builds - #: only; a release candidate refuses it). Unset, it follows the BRMA mode: - #: ``brma_list`` with area BRMAs, ``none`` otherwise. - private_rent_localisation: str | None = None - #: ``area`` sets each placed household's Rural Fuel Duty Relief flag off - #: its assigned output area (microcosm#1418); ``none`` leaves the engine's - #: default (measurement builds only; a release candidate refuses it). - #: Unset, the mode follows the geography assignment: ``area`` on the - #: atomic assignment, ``none`` on the legacy one. - rfdr_localisation: str | None = None def __post_init__(self) -> None: from .graph_population import GEOGRAPHY_ASSIGNMENTS @@ -120,71 +95,6 @@ def __post_init__(self) -> None: raise ValueError( f"Geography assignment must be one of {GEOGRAPHY_ASSIGNMENTS}." ) - from .graph_population import PROPERTY_LOCALISATION_MODES - - if self.property_localisation is None: - object.__setattr__( - self, - "property_localisation", - "price_paid" if self.geography_assignment == "atomic" else "none", - ) - if self.property_localisation not in PROPERTY_LOCALISATION_MODES: - raise ValueError( - f"Property localisation must be one of {PROPERTY_LOCALISATION_MODES}." - ) - if ( - self.property_localisation != "none" - and self.geography_assignment != "atomic" - ): - raise ValueError( - "Property localisation needs the atomic geography assignment." - ) - from .graph_population import BRMA_LOCALISATION_MODES - - if self.brma_localisation is None: - object.__setattr__( - self, - "brma_localisation", - "area" if self.geography_assignment == "atomic" else "none", - ) - if self.brma_localisation not in BRMA_LOCALISATION_MODES: - raise ValueError( - f"BRMA localisation must be one of {BRMA_LOCALISATION_MODES}." - ) - if self.brma_localisation != "none" and self.geography_assignment != "atomic": - raise ValueError("Area BRMAs need the atomic geography assignment.") - from .graph_population import PRIVATE_RENT_LOCALISATION_MODES - - if self.private_rent_localisation is None: - object.__setattr__( - self, - "private_rent_localisation", - "brma_list" if self.brma_localisation == "area" else "none", - ) - if self.private_rent_localisation not in PRIVATE_RENT_LOCALISATION_MODES: - raise ValueError( - "Private rent localisation must be one of " - f"{PRIVATE_RENT_LOCALISATION_MODES}." - ) - if ( - self.private_rent_localisation != "none" - and self.brma_localisation != "area" - ): - raise ValueError("Private rents from the lists need the area BRMAs.") - from .graph_population import RFDR_LOCALISATION_MODES - - if self.rfdr_localisation is None: - object.__setattr__( - self, - "rfdr_localisation", - "area" if self.geography_assignment == "atomic" else "none", - ) - if self.rfdr_localisation not in RFDR_LOCALISATION_MODES: - raise ValueError( - f"RFDR localisation must be one of {RFDR_LOCALISATION_MODES}." - ) - if self.rfdr_localisation != "none" and self.geography_assignment != "atomic": - raise ValueError("Area RFDR flags need the atomic geography assignment.") if not isinstance(self.source_vintage, str) or not self.source_vintage: raise ValueError("The FRS source vintage must be a non-empty string.") validate_sample_fraction(self.sample_fraction, label="UK full pool") @@ -212,11 +122,6 @@ def __post_init__(self) -> None: ) if len(set(self.geography_levels)) != len(self.geography_levels): raise ValueError("Geographic target levels must not repeat.") - if "brma" in self.geography_levels and self.brma_localisation != "area": - raise ValueError( - "BRMA targets need the area BRMAs: without them the rows " - "would bind on the spine's regional BRMA draw." - ) if self.target_families is not None and ( not self.target_families or any(not isinstance(f, str) or not f for f in self.target_families) @@ -306,14 +211,6 @@ def uk_full_graph( geography_assignment=config.geography_assignment, atomic_geography_definition=atomic_geography_definition, source_vintage=config.source_vintage, - property_localisation_mode=config.property_localisation, - brma_localisation_mode=config.brma_localisation, - private_rent_localisation_mode=config.private_rent_localisation, - rfdr_localisation_mode=config.rfdr_localisation, - # A sampled rung's rent node may rank a region on Great Britain's FRS - # renters; a full-scale build refuses (microcosm#1305 review). - full_scale=config.sample_fraction == 1.0 - and config.source_sample_fraction == 1.0, ) graph = append_uk_target_nodes( graph, @@ -326,7 +223,6 @@ def uk_full_graph( target_weight_rule=config.calibration.target_weight_rule, optional_sources=optional_target_sources, review_date=review_date, - brma_rows=config.brma_localisation == "area", ) cells = population_columns(graph, "uk.full.expand") # This structural checkpoint depends on every pool operation, including @@ -342,9 +238,7 @@ def uk_full_graph( structural=StructuralDelta.FILTER, base="uk.full.expand", inputs=population_slices(cells), - artifact_inputs=uk_pool_validation_inputs( - config.geography_assignment, config.property_localisation - ), + artifact_inputs=uk_pool_validation_inputs(config.geography_assignment), description="Checkpoint the complete geographic pool and selected-target problem.", ), ), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index b432fab704..44d66a9aed 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -7,12 +7,10 @@ from __future__ import annotations -import hashlib import importlib.util import json from collections.abc import Mapping from dataclasses import asdict, replace -from pathlib import Path import numpy as np import pandas as pd @@ -42,22 +40,17 @@ ) from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token -from microcosm.graph.randomness import keyed_uniform from microcosm.graph.weight_update import weight_update_receipt from .. import frame_sampling, gates from . import ( atomic_area_support, atomic_household_identity, - brma_area_layer, geographic_clones, geography_ladder, local_authority_input, national_frame, national_sampling, - private_rent_localisation, - property_localisation, - rfdr_area_layer, rowwise_dataset, rowwise_geography, spi_support, @@ -70,7 +63,6 @@ load_uk_oa_ladder, ) from .household_lineage import ( - HIGH_VALUE_SUPPORT_COPY_COLUMN, UK_HOUSEHOLD_LINEAGE_COLUMNS, UK_OPTIONAL_LINEAGE_COLUMNS, uk_household_clone_paths, @@ -83,7 +75,6 @@ verify_local_authority_engine_domain, ) from .national_frame import UK_NATIONAL_SCHEMA -from .receipt_counts import rounded_count, source_household_ids from .rowwise_dataset import expand_uk_geographic_pool POPULATION_RECEIPT_TYPE = ArtifactType("microcosm.uk.population-receipt", 1) @@ -98,34 +89,6 @@ #: ``uk.full.expand`` writes the index. Calibration K, weights and remapped ids #: are not inputs. UK_GEOGRAPHY_IDENTITY_INPUTS = UK_HOUSEHOLD_LINEAGE_COLUMNS -PROPERTY_LOCALISATION_MODES = property_localisation.PROPERTY_LOCALISATION_MODES -PROPERTY_QUANTILE_TYPE = ArtifactType("microcosm.uk.property-quantile-draw", 1) -#: Household columns the post-geography property nodes read and rewrite. -_PROPERTY_VALUE_COLUMNS = ( - "region", - "tenure_type", - "main_residence_value", - "property_wealth", -) -_PROPERTY_BAND_COLUMNS = ("council_tax_band",) -BRMA_LOCALISATION_MODES = brma_area_layer.BRMA_LOCALISATION_MODES -#: Household columns the post-geography BRMA node reads (microcosm#1262). -_BRMA_AREA_COLUMNS = ("atomic_area_system", "brma", "lsoa_code", "oa_code", "region") -PRIVATE_RENT_LOCALISATION_MODES = ( - private_rent_localisation.PRIVATE_RENT_LOCALISATION_MODES -) -RFDR_LOCALISATION_MODES = rfdr_area_layer.RFDR_LOCALISATION_MODES -#: Household columns the post-geography RFDR node reads (microcosm#1418). -_RFDR_AREA_COLUMNS = ("atomic_area_code", "atomic_area_system") -#: Household columns the post-geography private-rent node reads (microcosm#1262). -_PRIVATE_RENT_COLUMNS = ( - "brma", - "household_support_channel", - "num_bedrooms", - "region", - "rent", - "tenure_type", -) def context_frame(context: KernelContext) -> Frame: @@ -501,208 +464,6 @@ def run(self, context: KernelContext) -> KernelResult: ) -class UKPropertyValueKernel(_PopulationKernel): - """Read each owner clone's home value off its constituency's distribution. - - microcosm#1240 fixes 3 and 5. Every pool clone of one spine household - keeps one uniform (its weighted rank among the owner households of its - FRS region, ties broken by a keyed draw; high-value support copies take a - keyed draw inside the top stratum), and reads its value at that uniform - from the assigned constituency's uprated latest-sale knots. Weights are - untouched; property wealth moves by the value's change. - """ - - ref = "uk.full.property.value@1" - capabilities = Capabilities( - determinism=Determinism.SEEDED, - seed_source=SeedSource.KEYED, - numeric=Numeric.PLATFORM_BITWISE, - ) - - def implementation_hash(self) -> str: - from microcosm.frame import concepts as frame_concepts - - from .. import source_lineage - from . import household_lineage, tenure_constants - - # The spine keys come through the lineage readers and the clone index - # column geographic_clones names, the owner mask through - # tenure_constants' tenure categories and the owner uniforms through - # keyed draws; the spine keys and the quantile artifact are encoded - # canonically (microcosm#931). - return source_hash( - type(self), - property_localisation, - household_lineage, - atomic_household_identity, - source_lineage, - frame_concepts, - rowwise_dataset, - geographic_clones, - tenure_constants, - keyed_uniform, - canonical_json, - ) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - weights = np.asarray(context.weights["household"].values, dtype=float) - knots = property_localisation.load_value_knots( - Path(context.sources[property_localisation.QUANTILES_SOURCE]).read_bytes() - ) - stream = tuple(context.params["stream"]) - keys = property_localisation.spine_household_keys( - household, - source=str(context.params["source"]), - source_vintage=str(context.params["source_vintage"]), - ) - support = ( - household[HIGH_VALUE_SUPPORT_COPY_COLUMN].to_numpy().astype(bool) - if HIGH_VALUE_SUPPORT_COPY_COLUMN in household - else None - ) - u, uniform_receipt = property_localisation.owner_uniforms( - spine_key=keys, - region=household["region"].to_numpy(), - tenure_type=household["tenure_type"].to_numpy(), - # The spine's regional values (microcosm#1112) keep the value they - # replaced; ranking on it keeps this node's draws unchanged. - value=property_localisation.main_residence_rank_basis( - household, - declared=bool(context.params.get("rank_basis_declared", False)), - ), - weight=weights, - stream=stream, - support_copy=support, - support_stratum_lower=float(context.params["support_stratum_lower"]), - ) - new_value, new_wealth, receipt = property_localisation.localise_values( - u=u, - constituency_code=household["constituency_code"].to_numpy(), - tenure_type=household["tenure_type"].to_numpy(), - value=household["main_residence_value"].to_numpy(), - property_wealth=household["property_wealth"].to_numpy(), - knots=knots, - ) - index = pd.Index(household["household_id"], name="household_id") - household_ids = [int(value) for value in household["household_id"].to_numpy()] - payload = { - "schema": "microcosm.uk.property-quantile-draw.v1", - "household_ids": household_ids, - "u": [float(value) for value in u], - "cap": knots.cap, - "stream": list(stream), - "method": property_localisation.METHOD_VERSION, - "knots_inputs_sha256": knots.inputs_sha256, - "basis_year": knots.basis_year, - "rows_value_rewritten": receipt["rows_rewritten"], - } - return KernelResult( - columns={ - ("household", "main_residence_value"): pd.Series( - new_value, index=index, name="main_residence_value", dtype="float64" - ), - ("household", "property_wealth"): pd.Series( - new_wealth, index=index, name="property_wealth", dtype="float64" - ), - }, - artifacts={"quantiles": canonical_json(payload)}, - receipt={ - **receipt, - "uniforms": uniform_receipt, - "weights": "unchanged", - "household_mass": float(weights.sum()), - "knots_inputs_sha256": knots.inputs_sha256, - "basis_year": knots.basis_year, - "stream": list(stream), - }, - ) - - -class UKPropertyBandKernel(_PopulationKernel): - """Draw each England-and-Wales clone's council-tax band from its LSOA's VOA mix. - - microcosm#1240 fix 4: the band is read at the same uniform the value node - placed the home value with, so a clone's value and band agree by - construction. Rows outside England and Wales keep the spine's band. - """ - - ref = "uk.full.property.band@1" - capabilities = Capabilities( - determinism=Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE - ) - - def implementation_hash(self) -> str: - from . import tenure_constants - - # property_localisation's owner mask reads tenure_constants' tenure - # categories (microcosm#931). - return source_hash(type(self), property_localisation, tenure_constants) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - payload = json.loads(context.artifacts["quantiles"].payload) - if payload.get("schema") != "microcosm.uk.property-quantile-draw.v1": - raise ValueError( - "Property band node received an unknown quantile artifact." - ) - ids = [int(value) for value in household["household_id"].to_numpy()] - if payload["household_ids"] != ids: - raise ValueError( - "Property quantile draws do not align with the pool households." - ) - u = np.asarray(payload["u"], dtype=float) - mix = property_localisation.load_band_mix( - Path(context.sources[property_localisation.BAND_MIX_SOURCE]).read_bytes() - ) - scotland_mix = property_localisation.load_scotland_band_mix( - Path( - context.sources[property_localisation.SCOTLAND_BAND_MIX_SOURCE] - ).read_bytes() - ) - band_dtype = household["council_tax_band"].dtype - tenure = household["tenure_type"].to_numpy() - new_band, receipt = property_localisation.localise_bands( - u=u, - lsoa_code=household["lsoa_code"].to_numpy(), - band=household["council_tax_band"].to_numpy(), - tenure_type=tenure, - mix=mix, - scotland_mix=scotland_mix, - local_authority_code=household["local_authority_code"].to_numpy(), - ) - lsoa = household["lsoa_code"].to_numpy().astype(str) - area = household["local_authority_code"].to_numpy().astype(str) - owner = property_localisation.owner_mask(tenure) - england_wales = np.char.startswith(lsoa, "E") | np.char.startswith(lsoa, "W") - scotland = np.char.startswith(area, "S12") & ~england_wales - # Each mix is monotone within its area: an LSOA in England and Wales, - # a council area in Scotland. - violations = property_localisation.monotone_violations( - u[england_wales & owner], - new_band[england_wales & owner], - lsoa[england_wales & owner], - ) + property_localisation.monotone_violations( - u[scotland & owner], new_band[scotland & owner], area[scotland & owner] - ) - if violations: - raise ValueError("Property band draws are not monotone in the uniform.") - index = pd.Index(household["household_id"], name="household_id") - return KernelResult( - columns={ - ("household", "council_tax_band"): pd.Series( - new_band, index=index, name="council_tax_band" - ).astype(band_dtype), - }, - receipt={ - **receipt, - "monotone_violations": violations, - "band_mix_sha256": mix.source_sha256, - "scotland_band_mix_facts_sha256": scotland_mix.source_sha256, - }, - ) - - class UKLocalAuthorityResolveKernel(_PopulationKernel): """Resolve the engine's ``local_authority`` input from the derived code. @@ -739,236 +500,6 @@ def run(self, context: KernelContext) -> KernelResult: ) -class UKBRMAAreaKernel(_PopulationKernel): - """Read each placed Great Britain household's BRMA off its assigned area. - - microcosm#1262: the spine's ``frs_brma`` stage draws a BRMA within the - household's FRS region before any household has an area, so every clone - carried one BRMA wherever it landed. The committed layer places every - England-and-Wales LSOA and Scottish output area in one BRMA, so a clone's - BRMA, LHA rate, local authority and constituency describe one place. - Northern Ireland has no open BRMA geography and keeps the spine's draw. - """ - - ref = "uk.full.geography.brma@1" - capabilities = Capabilities( - determinism=Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE - ) - - def implementation_hash(self) -> str: - from . import receipt_counts - - # The receipt rounds its counts in receipt_counts (microcosm#931). - return source_hash(type(self), brma_area_layer, receipt_counts) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - payload = Path( - context.sources[brma_area_layer.BRMA_AREA_LAYER_SOURCE] - ).read_bytes() - layer = brma_area_layer.load_brma_area_layer(payload) - brma, counts = brma_area_layer.area_brmas( - layer=layer, - area_system=household["atomic_area_system"].to_numpy(), - lsoa_code=household["lsoa_code"].to_numpy(), - oa_code=household["oa_code"].to_numpy(), - region=household["region"].to_numpy(), - brma=household["brma"].to_numpy(), - ) - index = pd.Index(household["household_id"], name="household_id") - return KernelResult( - columns={ - ("household", "brma"): pd.Series(brma, index=index, name="brma").astype( - household["brma"].dtype - ), - }, - receipt={ - "rows_by_grain": { - grain: rounded_count(count) - for grain, count in sorted(counts.items()) - if grain != "changed" - }, - "rows_changed": rounded_count(counts["changed"]), - "layer_sha256": hashlib.sha256(payload).hexdigest(), - "method": layer.method, - }, - ) - - -class UKRFDRAreaKernel(_PopulationKernel): - """Set each placed household's Rural Fuel Duty Relief flag off its output area. - - microcosm#1418: policyengine-uk relieves a household's petrol and diesel - duty when ``in_rural_fuel_duty_relief_area`` is true. The committed layer - lists the Great Britain output areas in the scheme (HMRC Notice 2001's - islands, post town and postcode districts, by the ONS postcode lookup), so - a clone in one of them carries the flag. Northern Ireland has no area in - the scheme. - """ - - ref = "uk.full.geography.rfdr@1" - capabilities = Capabilities( - determinism=Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE - ) - - def implementation_hash(self) -> str: - from . import receipt_counts - - # The receipt rounds its counts in receipt_counts (microcosm#931). - return source_hash(type(self), rfdr_area_layer, receipt_counts) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - payload = Path( - context.sources[rfdr_area_layer.RFDR_AREA_LAYER_SOURCE] - ).read_bytes() - layer = rfdr_area_layer.load_rfdr_area_layer(payload) - strict = bool(context.params.get("strict_flags", True)) - flags, counts = rfdr_area_layer.area_flags( - layer=layer, - area_system=household["atomic_area_system"].to_numpy(), - area_code=household["atomic_area_code"].to_numpy(), - require_flagged=strict, - ) - index = pd.Index(household["household_id"], name="household_id") - column = rfdr_area_layer.RFDR_COLUMN - return KernelResult( - columns={ - ("household", column): pd.Series(flags, index=index, name=column), - }, - receipt={ - "rows_by_system": { - system: rounded_count(count) - for system, count in sorted(counts.items()) - if system != "flagged" - }, - "rows_flagged": rounded_count(counts["flagged"]), - "zero_flag_refusal": strict, - "layer_sha256": hashlib.sha256(payload).hexdigest(), - "method": layer.method, - }, - ) - - -class UKPrivateRentKernel(_PopulationKernel): - """Read each Great Britain private renter's rent off its BRMA's list of rents. - - microcosm#1262: the spine's rent describes the source household's FRS - region. Every private renter with a positive rent, in either channel, - keeps its rank among the FRS renters of its region and bedroom size (one - keyed tie-break per spine household, shared by all of its pool clones) - and reads that quantile off its assigned BRMA's list for homes of its - size, scaled so the region's mean rent is the FRS level on the census - private-renter mix. Placement follows all households, so at design - weights the level moves with the gap between the two mixes, which - calibration's tenure rows close. Weights are untouched; owners, social - renters, zero rents and Northern Ireland keep their rents bit for bit. The - region scales and ranks stay inside the kernel; the receipt carries - rounded record counts only. - """ - - ref = "uk.full.private_rent@1" - capabilities = Capabilities( - determinism=Determinism.SEEDED, - seed_source=SeedSource.KEYED, - numeric=Numeric.PLATFORM_BITWISE, - ) - - def implementation_hash(self) -> str: - from microcosm.frame import concepts as frame_concepts - - from .. import source_lineage - from . import household_lineage, receipt_counts - - # The spine keys come through the lineage readers and the clone index - # column geographic_clones names, and are encoded canonically; the - # tie-breaks are keyed draws; the receipt counts source households on - # national_frame's schema and rounds them in receipt_counts - # (microcosm#931). - return source_hash( - type(self), - private_rent_localisation, - property_localisation, - household_lineage, - atomic_household_identity, - source_lineage, - frame_concepts, - rowwise_dataset, - geographic_clones, - national_frame, - receipt_counts, - keyed_uniform, - canonical_json, - ) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - weights = np.asarray(context.weights["household"].values, dtype=float) - quantiles_payload = Path( - context.sources[private_rent_localisation.QUANTILES_SOURCE] - ).read_bytes() - counts_payload = Path( - context.sources[private_rent_localisation.COUNTS_SOURCE] - ).read_bytes() - quantiles = private_rent_localisation.load_rent_quantiles(quantiles_payload) - counts = private_rent_localisation.load_private_renter_counts(counts_payload) - stream = tuple(context.params["stream"]) - keys = property_localisation.spine_household_keys( - household, - source=str(context.params["source"]), - source_vintage=str(context.params["source_vintage"]), - ) - mask = private_rent_localisation.redraw_mask( - household["region"].to_numpy(), - household["tenure_type"].to_numpy(), - household["rent"].to_numpy(), - ) - redrawn = sorted({str(key) for key in keys[mask]}) - draws = keyed_uniform( - stream=stream, keys=[(key, "rent_rank") for key in redrawn] - ) - new_rent, details = private_rent_localisation.localise_rents( - spine_key=keys, - source=source_household_ids(household, stage="uk.full.private_rent"), - region=household["region"].to_numpy(), - tenure_type=household["tenure_type"].to_numpy(), - rent=household["rent"].to_numpy(), - num_bedrooms=household["num_bedrooms"].to_numpy(), - brma=household["brma"].to_numpy(), - channel=household["household_support_channel"].to_numpy(), - weight=weights, - tie_uniform=dict(zip(redrawn, (float(v) for v in draws), strict=True)), - quantiles=quantiles, - counts=counts, - strict=bool(context.params.get("strict_cells", True)), - ) - index = pd.Index(household["household_id"], name="household_id") - return KernelResult( - columns={ - ("household", "rent"): pd.Series( - new_rent, index=index, name="rent" - ).astype(household["rent"].dtype), - }, - receipt={ - "rows_redrawn": rounded_count(details["rows_redrawn"]), - "spine_households_redrawn": rounded_count( - details["spine_households_redrawn"] - ), - "cells_ranked_against_their_region": details["fallback_cells"], - "cells_ranked_against_great_britain": details["great_britain_cells"], - "regions_scaled": details["regions"], - "regions_scaled_on_great_britain": details[ - "regions_scaled_on_great_britain" - ], - "weights": "unchanged", - "quantiles_sha256": hashlib.sha256(quantiles_payload).hexdigest(), - "counts_sha256": hashlib.sha256(counts_payload).hexdigest(), - "method": private_rent_localisation.METHOD_VERSION, - "stream": list(stream), - }, - ) - - class UKPoolCheckpointKernel(_PopulationKernel): """Checkpoint the complete pool; refuse unless the shared atomic gate passed. @@ -998,27 +529,6 @@ def run(self, context: KernelContext) -> KernelResult: if validation.get("outcome") != "pass": raise ValueError("Shared atomic geography validation did not pass.") receipt["atomic_validation"] = validation - if "property_quantiles" in context.artifacts: - quantiles = json.loads(context.artifacts["property_quantiles"].payload) - household_ids = [ - int(value) - for value in context.tables["household"]["household_id"].to_numpy() - ] - if quantiles.get("household_ids") != household_ids: - raise ValueError( - "Property quantile draws do not align with the checkpointed pool." - ) - receipt["property_localisation"] = { - key: quantiles[key] - for key in ( - "cap", - "stream", - "method", - "knots_inputs_sha256", - "basis_year", - "rows_value_rewritten", - ) - } person = context.tables[UK_NATIONAL_SCHEMA.person_entity] id_column = UK_NATIONAL_SCHEMA.person_id_column ids = pd.Index(person[id_column].to_numpy(copy=True), name=id_column) @@ -1027,17 +537,15 @@ def run(self, context: KernelContext) -> KernelResult: ) -def uk_pool_validation_inputs( - geography_assignment: str, property_localisation_mode: str = "none" -) -> tuple[ArtifactInput, ...]: - """The typed edges that order the shared atomic gate (and the property draw) before the checkpoint.""" +def uk_pool_validation_inputs(geography_assignment: str) -> tuple[ArtifactInput, ...]: + """The typed edge that orders the shared atomic gate before the checkpoint.""" if geography_assignment != "atomic": return () from microcosm.build.graph_atomic_geography import ( ATOMIC_GEOGRAPHY_VALIDATION_TYPE, ) - inputs = ( + return ( ArtifactInput( "atomic_validation", "uk.full.geography.gate", @@ -1045,17 +553,6 @@ def uk_pool_validation_inputs( ATOMIC_GEOGRAPHY_VALIDATION_TYPE, ), ) - if property_localisation_mode == "price_paid": - inputs = ( - *inputs, - ArtifactInput( - "property_quantiles", - "uk.full.property.value", - "quantiles", - PROPERTY_QUANTILE_TYPE, - ), - ) - return inputs class UKGeographyGateKernel(_PopulationKernel): @@ -1100,19 +597,9 @@ def append_uk_population_nodes( atomic_geography_definition: Mapping | None = None, source_vintage: str | None = None, identity_source: str = "frs", - property_localisation_mode: str = "none", - brma_localisation_mode: str = "none", - private_rent_localisation_mode: str = "none", - rfdr_localisation_mode: str = "none", - full_scale: bool = True, ) -> Graph: """Compose sampling → replication → identity → assignment → derivation → gate. - ``property_localisation_mode="price_paid"`` (atomic assignment only) adds - the microcosm#1240 value and band nodes between the local-authority - resolve and the gates; ``"none"`` leaves the spine's values and bands on - every clone. - ``geography_assignment="atomic"`` (the default) keys every post-clone household by its lineage and runs the shared atomic-geography operators on the three UK support artifacts declared by ``atomic_geography_definition`` @@ -1138,26 +625,6 @@ def append_uk_population_nodes( raise ValueError("Atomic geography assignment requires the FRS vintage.") elif atomic_geography_definition is not None: raise ValueError("Legacy geography assignment takes no atomic definition.") - if property_localisation_mode not in PROPERTY_LOCALISATION_MODES: - raise ValueError( - f"Property localisation must be one of {PROPERTY_LOCALISATION_MODES}." - ) - if property_localisation_mode != "none" and geography_assignment != "atomic": - raise ValueError("Property localisation needs the atomic geography assignment.") - if brma_localisation_mode not in BRMA_LOCALISATION_MODES: - raise ValueError(f"BRMA localisation must be one of {BRMA_LOCALISATION_MODES}.") - if brma_localisation_mode != "none" and geography_assignment != "atomic": - raise ValueError("Area BRMAs need the atomic geography assignment.") - if private_rent_localisation_mode not in PRIVATE_RENT_LOCALISATION_MODES: - raise ValueError( - f"Private rent localisation must be one of {PRIVATE_RENT_LOCALISATION_MODES}." - ) - if private_rent_localisation_mode != "none" and brma_localisation_mode != "area": - raise ValueError("Private rents from the lists need the area BRMAs.") - if rfdr_localisation_mode not in RFDR_LOCALISATION_MODES: - raise ValueError(f"RFDR localisation must be one of {RFDR_LOCALISATION_MODES}.") - if rfdr_localisation_mode != "none" and geography_assignment != "atomic": - raise ValueError("Area RFDR flags need the atomic geography assignment.") national_sampling.validate_sample_fraction(sample_fraction, label="UK full build") national_sampling.validate_sample_seed(sample_seed, label="UK full build") common = {"time_period": str(time_period)} @@ -1286,11 +753,6 @@ def append_uk_population_nodes( definition=atomic_geography_definition, identity_source=identity_source, source_vintage=source_vintage, - property_localisation_mode=property_localisation_mode, - brma_localisation_mode=brma_localisation_mode, - private_rent_localisation_mode=private_rent_localisation_mode, - rfdr_localisation_mode=rfdr_localisation_mode, - full_scale=full_scale, ) nodes.extend(geography_nodes) declared = {s.name for s in graph.sources} @@ -1359,11 +821,6 @@ def _atomic_geography_nodes( definition: Mapping, identity_source: str, source_vintage: str, - property_localisation_mode: str = "none", - brma_localisation_mode: str = "none", - private_rent_localisation_mode: str = "none", - rfdr_localisation_mode: str = "none", - full_scale: bool = True, ) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: """Identity-keyed assignment on the shared operators (release path order).""" @@ -1432,39 +889,6 @@ def _atomic_geography_nodes( description="Resolve the engine local_authority input from the derived GSS code.", ) cells[("household", UK_LOCAL_AUTHORITY_INPUT_COLUMN)] = "string" - brma_nodes: tuple[Node, ...] = () - brma_sources: tuple[SourceRef, ...] = () - if brma_localisation_mode == "area": - brma_nodes, brma_sources = _brma_area_nodes(cells, common=common) - if private_rent_localisation_mode == "brma_list": - rent_nodes, rent_sources = _private_rent_nodes( - cells, - common=common, - seed=seed, - identity_source=identity_source, - source_vintage=source_vintage, - lineage_inputs=lineage_inputs, - strict=full_scale, - ) - brma_nodes = (*brma_nodes, *rent_nodes) - brma_sources = (*brma_sources, *rent_sources) - rfdr_nodes: tuple[Node, ...] = () - rfdr_sources: tuple[SourceRef, ...] = () - if rfdr_localisation_mode == "area": - rfdr_nodes, rfdr_sources = _rfdr_area_nodes( - cells, common=common, strict=full_scale - ) - property_nodes: tuple[Node, ...] = () - property_sources: tuple[SourceRef, ...] = () - if property_localisation_mode == "price_paid": - property_nodes, property_sources = _property_localisation_nodes( - cells, - common=common, - seed=seed, - identity_source=identity_source, - source_vintage=source_vintage, - lineage_inputs=lineage_inputs, - ) # The shared gate's typed validation artifact is platform-scoped, so it is # consumed by the pool checkpoint (uk.full.pool, no typed outputs), not by # this distribution gate, whose own artifact feeds the bitwise UK chain. @@ -1491,304 +915,8 @@ def _atomic_geography_nodes( ) for system in atomic_area_support.SYSTEMS ), - *property_sources, - *brma_sources, - *rfdr_sources, - ) - return ( - identity, - *shared[:-1], - local_authority, - *brma_nodes, - *rfdr_nodes, - *property_nodes, - shared[-1], - gate, - ), sources - - -def _property_localisation_nodes( - cells: dict, - *, - common: Mapping, - seed: int, - identity_source: str, - source_vintage: str, - lineage_inputs: tuple[str, ...], -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The microcosm#1240 value and band nodes, after derivation and before the gates.""" - for column in (*_PROPERTY_VALUE_COLUMNS, *_PROPERTY_BAND_COLUMNS): - if ("household", column) not in cells: - raise ValueError( - f"Property localisation needs the household column {column!r} on the pool." - ) - for column in ("constituency_code", "lsoa_code", "local_authority_code"): - if ("household", column) not in cells: - raise ValueError( - f"Property localisation runs after geography derives {column!r}." - ) - stream = ( - "sha256-u53-v1", - property_localisation.PROPERTY_STREAM_EXPERIMENT, - 0, - int(seed), - ) - # The regional home values stage (microcosm#1112) keeps the value it - # replaced; the node must receive it to rank owners as it did before. - rank_basis_declared = ( - "household", - property_localisation.RANK_BASIS_COLUMN, - ) in cells - rank_basis_inputs = ( - (property_localisation.RANK_BASIS_COLUMN,) if rank_basis_declared else () - ) - value = Node( - id="uk.full.property.value", - kernel=UKPropertyValueKernel.ref, - population="uk.full.expand", - inputs=( - Slice( - "household", - tuple( - sorted( - { - *lineage_inputs, - *_PROPERTY_VALUE_COLUMNS, - *rank_basis_inputs, - "constituency_code", - } - ) - ), - ), - ), - outputs=( - Owned( - "household", - "main_residence_value", - cells[("household", "main_residence_value")], - rewrite=True, - ), - Owned( - "household", - "property_wealth", - cells[("household", "property_wealth")], - rewrite=True, - ), - ), - sources=(property_localisation.QUANTILES_SOURCE,), - params={ - **common, - "source": identity_source, - "source_vintage": source_vintage, - "stream": stream, - "support_stratum_lower": property_localisation.SUPPORT_STRATUM_LOWER, - "method": property_localisation.METHOD_VERSION, - "rank_basis_declared": rank_basis_declared, - }, - artifact_outputs=(ArtifactOutput("quantiles", PROPERTY_QUANTILE_TYPE),), - description="Read each owner clone's home value off its constituency's uprated sale distribution at its region rank.", - ) - band = Node( - id="uk.full.property.band", - kernel=UKPropertyBandKernel.ref, - population="uk.full.expand", - inputs=( - Slice( - "household", - tuple( - sorted( - { - *_PROPERTY_BAND_COLUMNS, - "local_authority_code", - "lsoa_code", - "tenure_type", - } - ) - ), - ), - ), - outputs=( - Owned( - "household", - "council_tax_band", - cells[("household", "council_tax_band")], - rewrite=True, - ), - ), - sources=( - property_localisation.BAND_MIX_SOURCE, - property_localisation.SCOTLAND_BAND_MIX_SOURCE, - ), - params={**common, "method": property_localisation.METHOD_VERSION}, - artifact_inputs=( - ArtifactInput("quantiles", value.id, "quantiles", PROPERTY_QUANTILE_TYPE), - ), - description=( - "Draw each England-and-Wales owner clone's council-tax band from its " - "LSOA's VOA mix, and each Scottish owner clone's from its council " - "area's CTAXBASE mix, at the same uniform." - ), - ) - sources = ( - SourceRef( - property_localisation.QUANTILES_SOURCE, - "raw-bytes-v1", - "Per-constituency quantile knots of the uprated Land Registry sale values.", - ), - SourceRef( - property_localisation.BAND_MIX_SOURCE, - "raw-bytes-v1", - "Per-LSOA cumulative VOA council-tax band shares.", - ), - SourceRef( - property_localisation.SCOTLAND_BAND_MIX_SOURCE, - "raw-bytes-v1", - "Per-council-area chargeable dwellings by council-tax band " - "(Scottish Government CTAXBASE), vendored from Chronicle.", - ), - ) - return (value, band), sources - - -def _brma_area_nodes( - cells: dict, *, common: Mapping -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The microcosm#1262 area-BRMA node, after the local-authority resolve.""" - - for column in _BRMA_AREA_COLUMNS: - if ("household", column) not in cells: - raise ValueError( - f"The area BRMA node needs the household column {column!r} on the pool." - ) - node = Node( - id="uk.full.geography.brma", - kernel=UKBRMAAreaKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", tuple(sorted(_BRMA_AREA_COLUMNS))),), - outputs=( - Owned("household", "brma", cells[("household", "brma")], rewrite=True), - ), - sources=(brma_area_layer.BRMA_AREA_LAYER_SOURCE,), - params={**common, "method": brma_area_layer.METHOD_VERSION}, - description="Read each placed Great Britain household's BRMA off its assigned area.", - ) - source = SourceRef( - brma_area_layer.BRMA_AREA_LAYER_SOURCE, - "raw-bytes-v1", - "BRMA of every England-and-Wales LSOA and Scottish output area.", - ) - return (node,), (source,) - - -def _rfdr_area_nodes( - cells: dict, *, common: Mapping, strict: bool = True -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The microcosm#1418 area RFDR node, after the local-authority resolve. - - ``strict`` (the build's full scale) keeps the refusal of a support system - none of whose households is flagged, as the private-rent node keeps its - strict cells; malformed codes refuse at every scale. - """ - - for column in _RFDR_AREA_COLUMNS: - if ("household", column) not in cells: - raise ValueError( - f"The area RFDR node needs the household column {column!r} on the pool." - ) - column = rfdr_area_layer.RFDR_COLUMN - node = Node( - id="uk.full.geography.rfdr", - kernel=UKRFDRAreaKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", _RFDR_AREA_COLUMNS),), - outputs=(Owned("household", column, "bool"),), - sources=(rfdr_area_layer.RFDR_AREA_LAYER_SOURCE,), - params={ - **common, - "method": rfdr_area_layer.METHOD_VERSION, - "strict_flags": bool(strict), - }, - description="Set each placed household's Rural Fuel Duty Relief flag off its area.", - ) - cells[("household", column)] = "bool" - source = SourceRef( - rfdr_area_layer.RFDR_AREA_LAYER_SOURCE, - "raw-bytes-v1", - "Great Britain output areas in the Rural Fuel Duty Relief scheme.", - ) - return (node,), (source,) - - -def _private_rent_nodes( - cells: dict, - *, - common: Mapping, - seed: int, - identity_source: str, - source_vintage: str, - lineage_inputs: tuple[str, ...], - strict: bool = True, -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The microcosm#1262 private-rent node, after the area-BRMA node. - - ``strict`` holds at full scale; a sampled development rung relaxes it so - that a region its sample leaves without FRS renters ranks and scales on - Great Britain's (``private_rent_localisation.localise_rents``). - """ - - for column in _PRIVATE_RENT_COLUMNS: - if ("household", column) not in cells: - raise ValueError( - f"The private-rent node needs the household column {column!r} on the pool." - ) - stream = ( - "sha256-u53-v1", - private_rent_localisation.STREAM_EXPERIMENT, - 0, - int(seed), - ) - node = Node( - id="uk.full.private_rent", - kernel=UKPrivateRentKernel.ref, - population="uk.full.expand", - inputs=( - Slice( - "household", tuple(sorted({*lineage_inputs, *_PRIVATE_RENT_COLUMNS})) - ), - ), - outputs=( - Owned("household", "rent", cells[("household", "rent")], rewrite=True), - ), - sources=( - private_rent_localisation.QUANTILES_SOURCE, - private_rent_localisation.COUNTS_SOURCE, - ), - params={ - **common, - "source": identity_source, - "source_vintage": source_vintage, - "stream": stream, - "method": private_rent_localisation.METHOD_VERSION, - "strict_cells": bool(strict), - }, - description=( - "Read each Great Britain private renter's rent off its assigned BRMA's " - "list at its FRS rank, scaled to its region's FRS level." - ), - ) - sources = ( - SourceRef( - private_rent_localisation.QUANTILES_SOURCE, - "raw-bytes-v1", - "Weekly rent quantiles by BRMA and LHA category B to E.", - ), - SourceRef( - private_rent_localisation.COUNTS_SOURCE, - "raw-bytes-v1", - "Census private-rented households by FRS region, BRMA and bedrooms.", - ), ) - return (node,), sources + return (identity, *shared[:-1], local_authority, shared[-1], gate), sources def register_uk_population_kernels(registry: KernelRegistry) -> None: @@ -1801,12 +929,7 @@ def register_uk_population_kernels(registry: KernelRegistry) -> None: UKSampleNormalizationKernel(), UKGeographicExpansionKernel(), UKGeographyIdentityKernel(), - UKPropertyValueKernel(), - UKPropertyBandKernel(), UKLocalAuthorityResolveKernel(), - UKBRMAAreaKernel(), - UKRFDRAreaKernel(), - UKPrivateRentKernel(), UKLocationDrawKernel(), UKGeographyMappingKernel(), UKGeographyGateKernel(), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index e83b78397f..6e15957c07 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -457,16 +457,6 @@ def run(self, context: KernelContext) -> KernelResult: or f"{spec.family}/{target_geography(spec)}" in families ) ) - brma_left_out = 0 - if context.params.get("brma_rows", True) is False: - # Without area BRMAs the BRMA rows would bind on the spine's - # regional draw, unrelated to placement (microcosm#1305 review). - brma_left_out = sum( - target_geography(spec) == "brma" for spec in registry.specs - ) - registry = registry.select( - predicate=lambda spec: target_geography(spec) != "brma" - ) selected = select_targets( registry, geography_levels=levels, geography_resolver=target_geography ) @@ -476,19 +466,6 @@ def run(self, context: KernelContext) -> KernelResult: "registry": registry_payload(selected.registry), "receipt": selected.receipt, "target_families": None if families is None else list(families), - **( - {} - if context.params.get("brma_rows", True) - else { - "brma_rows_left_out": { - "rows": brma_left_out, - "reason": ( - "no area BRMAs (--brma-localisation none): the rows " - "would bind on the spine's regional BRMA draw" - ), - } - } - ), } return KernelResult(artifacts={"selection": canonical_json(payload)}) @@ -903,14 +880,8 @@ def append_uk_target_nodes( target_weight_rule: str = "uniform", optional_sources: tuple[str, ...] = (), review_date: str | None = None, - brma_rows: bool = True, ) -> Graph: - """Default all geographies. Scope is independent of K, k and solver outcomes. - - ``brma_rows=False`` (no area BRMAs) leaves the BRMA-grain rows out of the - selection: they bind on each placed household's own BRMA, and the spine's - regional draw is unrelated to placement (microcosm#1305 review). - """ + """Default all geographies. Scope is independent of K, k and solver outcomes.""" if geography_levels is not None and not geography_levels: raise ValueError("An explicit geography selector must contain levels.") @@ -951,7 +922,6 @@ def append_uk_target_nodes( params={ "geography_levels": geography_levels, "target_families": target_families, - **({} if brma_rows else {"brma_rows": False}), }, artifact_inputs=(surface,), artifact_outputs=(ArtifactOutput("selection", TARGET_SELECTION_TYPE),), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py index fb893beeb3..39419f36a8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py @@ -394,11 +394,6 @@ def checkpoint_references(sidecar_path: Path | None) -> tuple[list[dict], str | "uk.full.geography.assign", "uk.full.geography.derive", "uk.full.geography.local_authority", - "uk.full.geography.brma", - "uk.full.geography.rfdr", - "uk.full.private_rent", - "uk.full.property.value", - "uk.full.property.band", "uk.full.geography.gate", ), "targets": ( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_rent_localisation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_rent_localisation.py index 85f74904d7..04c64b4f87 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_rent_localisation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_rent_localisation.py @@ -2,22 +2,23 @@ The UK spine carries each private renter's rent from its source household (the FRS report, or the SPI housing shell's draw from FRS donors), which -describes its FRS region, not the district the household is placed in. After -geography, ``uk.full.private_rent`` keeps each private renter's rank among -the FRS renters of its region and bedroom size and reads that quantile off -the list of rents of the BRMA its assigned area sits in -(``uk/brma_rent_quantiles.json``), scaled per FRS region so that, on the -census private-renter mix of the region's BRMAs, its mean rent is the FRS level -(María's rulings of 2026-10-15/16, kept on 2026-10-17): the lists set where -rents sit within a region and their spread within a BRMA, not the level. -Placement follows all households, not renters, so at design weights a -region's mean rent moves by the gap between the two mixes until -calibration's tenure rows restore the renter mix; the receipts report that -level as a check. +describes its FRS region, not the district the household is placed in. The +clone-first spine's private-rent stage +(``spine_private_rent.UKPrivateRentStageTransform``, G7 of microcosm#931) +keeps each private renter's rank among the FRS renters of its region and +bedroom size and reads that quantile off the list of rents of the BRMA its +drawn area sits in (``uk/brma_rent_quantiles.json``), scaled per FRS region +so that, on the census private-renter mix of the region's BRMAs, its mean +rent is the FRS level (María's rulings of 2026-10-15/16, kept on +2026-10-17): the lists set where rents sit within a region and their spread +within a BRMA, not the level. Placement follows all households, not renters, +so at design weights a region's mean rent moves by the gap between the two +mixes until calibration's tenure rows restore the renter mix; the receipts +report that level as a check. Everything here is pure and depends on the standard library, numpy and -pandas only, so the build tool can load this module by path; the graph -kernel supplies the spine keys and the keyed tie-break uniforms. +pandas only, so the build tool can load this module by path; the spine's +private-rent stage supplies the row keys and the keyed tie-break uniforms. """ from __future__ import annotations @@ -36,7 +37,6 @@ "COUNTS_SOURCE", "GRID", "METHOD_VERSION", - "PRIVATE_RENT_LOCALISATION_MODES", "QUANTILES_KIND", "QUANTILES_METHOD", "QUANTILES_SCHEMA_VERSION", @@ -55,9 +55,6 @@ "rent_uniforms", ] -#: ``brma_list`` re-draws each Great Britain private renter's rent from its -#: assigned BRMA's list of rents; ``none`` keeps the spine's rents. -PRIVATE_RENT_LOCALISATION_MODES = ("none", "brma_list") METHOD_VERSION = "uk-private-rent-localisation-v1" STREAM_EXPERIMENT = "uk-post-clone-private-rent-v1" #: Graph source names: the committed rent quantiles and the census counts of diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py index 3068b51d60..6b33be0d43 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py @@ -39,7 +39,6 @@ "METHOD_VERSION", "RFDR_AREA_LAYER_SOURCE", "RFDR_COLUMN", - "RFDR_LOCALISATION_MODES", "SCHEMA_VERSION", "SYSTEMS", "RFDRAreaLayer", @@ -55,10 +54,7 @@ METHOD_VERSION = "uk-rfdr-area-layer-v1" #: Graph source name of the committed layer bytes. RFDR_AREA_LAYER_SOURCE = "uk_rfdr_area_layer" -#: ``area`` sets each placed Great Britain household's flag from its assigned -#: output area; ``none`` leaves the engine's default on every clone. -RFDR_LOCALISATION_MODES = ("none", "area") -#: The policyengine-uk household input the node writes. +#: The policyengine-uk household input the spine's fuel-relief stage sets. RFDR_COLUMN = "in_rural_fuel_duty_relief_area" #: The support systems the layer covers, by the output-area code they read. SYSTEMS: Mapping[str, str] = { diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py index 9df503c43a..30e781c75c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py @@ -1077,18 +1077,10 @@ def validate_cli_args(args: argparse.Namespace) -> None: if _geography_assignment(args) == "legacy": # The release line is the identity-keyed atomic assignment. refused.append("--geography-assignment legacy") - if getattr(args, "property_localisation", "price_paid") == "none": - # The release line localises home values and bands (microcosm#1240). - refused.append("--property-localisation none") - if getattr(args, "brma_localisation", "area") == "none": - # The release line reads BRMAs off the assigned area (microcosm#1262). - refused.append("--brma-localisation none") - if getattr(args, "private_rent_localisation", "brma_list") == "none": - # The release line re-draws private rents from the area's list. - refused.append("--private-rent-localisation none") - if getattr(args, "rfdr_localisation", "area") == "none": - # The release line flags its Rural Fuel Duty Relief areas (#1418). - refused.append("--rfdr-localisation none") + # A dense build takes its home values, BRMAs, private rents and + # fuel-relief flags from the spine, and spine_posture requires the + # spine stages that set them before the build pins or reads any other + # input (microcosm#931), so no flag turns them off. target_levels = getattr(args, "target_geographies", None) if target_levels is not None and not set(target_levels) & { "constituency", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py index e2b58966ea..fdecee56eb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/size_checkpoint.py @@ -133,46 +133,6 @@ def uk_size_checkpoint_identity( for system, source in ATOMIC_SUPPORT_SOURCES.items() }, } - # microcosm#1240: a localised run also binds the two property resources. - mode = getattr(args, "property_localisation", "none") - if mode == "price_paid": - from .property_localisation import BAND_MIX_SOURCE, QUANTILES_SOURCE - - identity["property_localisation"] = { - "mode": mode, - "resource_pins": { - name: dict(pins[name]) for name in (QUANTILES_SOURCE, BAND_MIX_SOURCE) - }, - } - # microcosm#1262: an area-BRMA run also binds the layer. - if getattr(args, "brma_localisation", "none") == "area": - from .brma_area_layer import BRMA_AREA_LAYER_SOURCE - - identity["brma_localisation"] = { - "mode": "area", - "resource_pins": { - BRMA_AREA_LAYER_SOURCE: dict(pins[BRMA_AREA_LAYER_SOURCE]) - }, - } - # microcosm#1418: an area-RFDR run also binds its layer. - if getattr(args, "rfdr_localisation", "none") == "area": - from .rfdr_area_layer import RFDR_AREA_LAYER_SOURCE - - identity["rfdr_localisation"] = { - "mode": "area", - "resource_pins": { - RFDR_AREA_LAYER_SOURCE: dict(pins[RFDR_AREA_LAYER_SOURCE]) - }, - } - if getattr(args, "private_rent_localisation", "none") == "brma_list": - from .private_rent_localisation import COUNTS_SOURCE, QUANTILES_SOURCE - - identity["private_rent_localisation"] = { - "mode": "brma_list", - "resource_pins": { - name: dict(pins[name]) for name in (QUANTILES_SOURCE, COUNTS_SOURCE) - }, - } return identity diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py b/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py index 5f94a8d902..d0e14e1897 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_full_graph_admission.py @@ -45,10 +45,6 @@ from microcosm.graph.canonical import canonical_json from test_support.microcosm_build.uk_atomic_support_fixtures import ( toy_support_sources, - write_toy_brma_area_layer, - write_toy_private_rent_resources, - write_toy_property_resources, - write_toy_rfdr_area_layer, write_toy_supports, ) from test_support.microcosm_build.uk_full_population_graph import ( @@ -216,10 +212,6 @@ def registry(): "uk_ladder": ladder_path, "uk_ledger_facts": ladder_path, **toy_support_sources(support_paths), - **write_toy_property_resources(tmp_path / "property"), - **write_toy_brma_area_layer(tmp_path / "brma"), - **write_toy_private_rent_resources(tmp_path / "rent"), - **write_toy_rfdr_area_layer(tmp_path / "rfdr"), } store = ContentStore(tmp_path / "store") checkpoint = compile_graph(_through(graph, "uk.full.gates.preflight")) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py index b5fc46d1ae..a7e41d7d0f 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py @@ -1075,7 +1075,8 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "hmrc_stamp_tax_facts.json", "hmrc_isa_statistics_facts.json", "hmrc_benefits_in_kind_facts.json", - # microcosm#1095: dense builds only, uk_runtime.property_localisation. + # microcosm#1095: no build reads it since the dense band node + # retired (#931); uk_runtime.property_localisation keeps its loader. "scotgov_chargeable_dwellings_by_band.json", # microcosm#1095 S2(a): frs_council_tax's Scottish level. "scotgov_band_d_liability_facts.json", @@ -1264,7 +1265,8 @@ def test_uk_package_loads(self) -> None: "hmrc_stamp_tax_facts.json", "hmrc_isa_statistics_facts.json", "hmrc_benefits_in_kind_facts.json", - # microcosm#1095: dense builds only, uk_runtime.property_localisation. + # microcosm#1095: no build reads it since the dense band node + # retired (#931); uk_runtime.property_localisation keeps its loader. "scotgov_chargeable_dwellings_by_band.json", # microcosm#1095 S2(a): frs_council_tax's Scottish level. "scotgov_band_d_liability_facts.json", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_brma_area_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_brma_area_graph.py deleted file mode 100644 index 1954bfb188..0000000000 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_brma_area_graph.py +++ /dev/null @@ -1,161 +0,0 @@ -"""The post-geography area-BRMA node in the UK full-build graph (microcosm#1262).""" - -from __future__ import annotations - -import pytest - -from microcosm.build.uk_runtime.atomic_area_support import ( - uk_atomic_assignment_definition, -) -from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig -from microcosm.build.uk_runtime.graph_population import _brma_area_nodes -from microcosm.graph import ContentStore, compile_graph, run_graph -from test_support.microcosm_build.uk_atomic_support_fixtures import ( - TOY_AREA_BRMAS, - toy_support_sources, - write_toy_brma_area_layer, - write_toy_supports, -) -from test_support.microcosm_build.uk_full_population_graph import ( - graph_and_registry, -) -from test_support.microcosm_build.uk_ladder_rowwise_clone import ( - toy_ladder as toy_ladder, -) - - -def _atomic(k: int, tmp_path, mode: str = "area"): - payloads, paths = write_toy_supports(tmp_path / "supports") - definition = uk_atomic_assignment_definition(payloads, seed=7) - graph, registry = graph_and_registry( - k, - geography_assignment="atomic", - definition=definition, - brma_localisation_mode=mode, - ) - return graph, registry, paths, definition - - -@pytest.mark.parametrize("k", [1, 3]) -def test_brma_node_reads_each_clone_area_and_replays(k, toy_ladder, tmp_path) -> None: - _, ladder_path = toy_ladder - graph, registry, paths, definition = _atomic(k, tmp_path) - compiled = compile_graph(graph) - order = compiled.order - # The node reads the derived area codes and precedes the geography gates. - assert order.index("uk.full.geography.derive") < order.index( - "uk.full.geography.brma" - ) - assert order.index("uk.full.geography.brma") < order.index("uk.full.geography.gate") - assert order.index("uk.full.geography.brma") < order.index("uk.full.geography_gate") - sources = { - "uk_ladder": ladder_path, - "fixture": ladder_path, - **toy_support_sources(paths), - **write_toy_brma_area_layer(tmp_path / "brma"), - } - store = ContentStore(tmp_path / "store") - first = run_graph(compiled, sources=sources, store=store, kernels=registry) - household = first.population("uk.full.expand").table("household") - receipt = first.nodes["uk.full.geography.brma"].receipt - system = household["atomic_area_system"].astype(str) - great_britain = system != "uk_ni_data_zone_2021" - for area_column, rows in ( - ("lsoa_code", system == "uk_ew_output_area_2021"), - ("oa_code", system == "uk_scotland_output_area_2022"), - ): - for code, brma in zip( - household.loc[rows, area_column].astype(str), - household.loc[rows, "brma"].astype(str), - strict=True, - ): - assert brma == TOY_AREA_BRMAS.get(code, "MAIDSTONE") - # Northern Ireland keeps the spine's regional draw. - assert set(household.loc[~great_britain, "brma"].astype(str)) == {"BELFAST"} - assert "MAIDSTONE" not in set(household.loc[great_britain, "brma"].astype(str)) - assert set(receipt["rows_by_grain"]) == { - "ew_lsoa21", - "northern_ireland", - "scotland_oa22", - } - _, fresh, _, _ = _atomic(k, tmp_path / "fresh") - replay = run_graph( - compiled, sources=sources, store=store, kernels=fresh, resume="require" - ) - assert all(node.hit for node in replay.nodes.values()) - - -def test_brma_follows_the_area_not_the_clone(toy_ladder, tmp_path) -> None: - """Every clone in one area gets that area's BRMA, whatever its source.""" - - _, ladder_path = toy_ladder - graph, registry, paths, _ = _atomic(3, tmp_path) - sources = { - "uk_ladder": ladder_path, - "fixture": ladder_path, - **toy_support_sources(paths), - **write_toy_brma_area_layer(tmp_path / "brma"), - } - run = run_graph( - compile_graph(graph), - sources=sources, - store=ContentStore(tmp_path / "store"), - kernels=registry, - ) - household = run.population("uk.full.expand").table("household") - per_area = household.groupby(household["oa_code"].astype(str))["brma"].nunique() - assert (per_area == 1).all() - - -def test_none_mode_adds_no_brma_node(tmp_path) -> None: - graph, _, _, _ = _atomic(1, tmp_path, mode="none") - assert "uk.full.geography.brma" not in compile_graph(graph).order - assert "uk_brma_area_layer" not in {source.name for source in graph.sources} - - -def test_area_brmas_need_the_atomic_assignment() -> None: - with pytest.raises(ValueError, match="atomic geography assignment"): - graph_and_registry( - 1, geography_assignment="legacy", brma_localisation_mode="area" - ) - with pytest.raises(ValueError, match="BRMA localisation must be one of"): - graph_and_registry(1, brma_localisation_mode="spread") - - -def test_the_node_refuses_a_pool_without_its_columns() -> None: - cells = { - ("household", column): "string" - for column in ("atomic_area_system", "lsoa_code", "oa_code", "region") - } - with pytest.raises(ValueError, match="'brma'"): - _brma_area_nodes(cells, common={"time_period": "2023"}) - - -def test_config_follows_the_geography_assignment() -> None: - assert UKFullBuildConfig(calibration_year=2025).brma_localisation == "area" - legacy = UKFullBuildConfig( - calibration_year=2025, - geography_assignment="legacy", - property_localisation="none", - ) - assert legacy.brma_localisation == "none" - with pytest.raises(ValueError, match="atomic geography assignment"): - UKFullBuildConfig( - calibration_year=2025, - geography_assignment="legacy", - property_localisation="none", - brma_localisation="area", - ) - - -def test_brma_targets_need_area_brmas(): - """microcosm#1305 review: without area BRMAs the rows would bind on the spine's draw.""" - assert UKFullBuildConfig( - calibration_year=2025, geography_levels=("la", "brma") - ).geography_levels == ("la", "brma") - with pytest.raises(ValueError, match="BRMA targets need the area BRMAs"): - UKFullBuildConfig( - calibration_year=2025, - geography_assignment="legacy", - geography_levels=("la", "brma"), - ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 30d1496c29..b0e031ac0e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -4602,94 +4602,6 @@ def test_solve_reuse_refuses_an_out_that_holds_a_bundle( assert _files(out) == before -def test_a_release_candidate_refuses_the_brma_and_rent_measurement_modes( - tmp_path, monkeypatch -): - """microcosm#1305 review: both none modes are for measurement builds only.""" - monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) - with pytest.raises(ValueError, match="--brma-localisation none"): - cli.validate_cli_args( - arguments(tmp_path, "--release-candidate", "--brma-localisation", "none") - ) - with pytest.raises(ValueError, match="--private-rent-localisation none"): - cli.validate_cli_args( - arguments( - tmp_path, "--release-candidate", "--private-rent-localisation", "none" - ) - ) - - -def test_a_release_candidate_refuses_the_rfdr_measurement_mode(tmp_path, monkeypatch): - """Anthony's review of microcosm#1420: the release line flags its Rural - Fuel Duty Relief areas; none is for measurement builds only.""" - monkeypatch.setenv(SIGNING_KEY_ENV, TEST_SIGNING_KEY) - with pytest.raises(ValueError, match="--rfdr-localisation none"): - cli.validate_cli_args( - arguments(tmp_path, "--release-candidate", "--rfdr-localisation", "none") - ) - - -def test_the_rfdr_layer_needs_area_mode_and_a_release_the_committed_layer(tmp_path): - """Anthony's review of microcosm#1420: an override nothing reads is refused, - and a release candidate builds on the committed layer only.""" - other = tmp_path / "other.json" - other.write_text("{}") - with pytest.raises(ValueError, match="need --rfdr-localisation area"): - cli._prepare_rfdr_localisation( - SimpleNamespace( - rfdr_localisation="none", - rfdr_area_layer=other, - rfdr_area_layer_sha256=None, - ), - {}, - {}, - ) - with pytest.raises(ValueError, match="requires the committed RFDR area layer"): - cli._prepare_rfdr_localisation( - SimpleNamespace( - rfdr_localisation="area", - rfdr_area_layer=other, - rfdr_area_layer_sha256=None, - release_candidate=True, - ), - {}, - {}, - ) - pins: dict = {} - sources: dict = {} - binding = cli._prepare_rfdr_localisation( - SimpleNamespace( - rfdr_localisation="area", - rfdr_area_layer=None, - rfdr_area_layer_sha256=None, - release_candidate=True, - ), - pins, - sources, - ) - assert binding["mode"] == "area" - assert set(pins) == set(sources) == {"uk_rfdr_area_layer"} - assert sources["uk_rfdr_area_layer"].name == "rfdr_area_layer.json" - assert binding["resource_pins"]["uk_rfdr_area_layer"] == pins["uk_rfdr_area_layer"] - - -@pytest.mark.parametrize( - ("assignment", "mode"), [("atomic", "area"), ("legacy", "none")] -) -def test_an_unset_rfdr_mode_follows_the_geography_assignment( - tmp_path, assignment, mode -): - """The independent review of microcosm#1420: a legacy-assignment build - through the CLI need not pass --rfdr-localisation none, as the build config - resolves it the same way.""" - args = arguments(tmp_path, "--geography-assignment", assignment) - assert args.rfdr_localisation is None - pins: dict = {} - binding = cli._prepare_rfdr_localisation(args, pins, {}) - assert binding["mode"] == args.rfdr_localisation == mode - assert bool(pins) is (mode == "area") - - def test_target_geographies_can_name_the_brma_rows(tmp_path): assert arguments( tmp_path, "--target-geographies", "la,brma" @@ -4701,59 +4613,6 @@ def test_target_geographies_can_name_the_brma_rows(tmp_path): arguments(tmp_path, "--target-geographies", "la,ward") -def test_the_brma_and_rent_resource_flags_need_their_modes(tmp_path): - """microcosm#1305 review: an override nothing reads is refused, not ignored.""" - other = tmp_path / "other.json" - other.write_text("{}") - with pytest.raises(ValueError, match="need --brma-localisation area"): - cli._prepare_brma_localisation( - SimpleNamespace( - brma_localisation="none", - brma_area_layer=other, - brma_area_layer_sha256=None, - ), - {}, - {}, - ) - with pytest.raises(ValueError, match="need --private-rent-localisation brma_list"): - cli._prepare_private_rent_localisation( - SimpleNamespace( - brma_localisation="none", - private_rent_localisation="none", - brma_rent_quantiles=None, - brma_rent_quantiles_sha256="0" * 64, - ), - {}, - {}, - ) - # A release candidate builds on the committed bytes only. - with pytest.raises(ValueError, match="requires the committed BRMA area layer"): - cli._prepare_brma_localisation( - SimpleNamespace( - brma_localisation="area", - brma_area_layer=other, - brma_area_layer_sha256=None, - release_candidate=True, - ), - {}, - {}, - ) - with pytest.raises( - ValueError, match="requires the committed 'uk_brma_rent_quantiles' resource" - ): - cli._prepare_private_rent_localisation( - SimpleNamespace( - brma_localisation="area", - private_rent_localisation="brma_list", - brma_rent_quantiles=other, - brma_rent_quantiles_sha256=None, - release_candidate=True, - ), - {}, - {}, - ) - - def test_the_thin_rows_only_switch_is_gone(tmp_path): """María's ruling of 2026-10-26 dropped the withholding, and with it --withhold-thin-rows-only; a script that still passes it is refused.""" @@ -4762,6 +4621,40 @@ def test_the_thin_rows_only_switch_is_gone(tmp_path): arguments(tmp_path, "--withhold-thin-rows-only") +#: The dense line's localisation flags and their resource overrides, retired +#: with the property, BRMA, private-rent and fuel-relief nodes (#931). +_RETIRED_LOCALISATION_FLAGS = ( + "--property-localisation", + "--price-paid-quantiles", + "--price-paid-quantiles-sha256", + "--voa-band-mix", + "--voa-band-mix-sha256", + "--scotgov-band-mix", + "--scotgov-band-mix-sha256", + "--brma-localisation", + "--brma-area-layer", + "--brma-area-layer-sha256", + "--private-rent-localisation", + "--brma-rent-quantiles", + "--brma-rent-quantiles-sha256", + "--rfdr-localisation", + "--rfdr-area-layer", + "--rfdr-area-layer-sha256", +) + + +@pytest.mark.parametrize("flag", _RETIRED_LOCALISATION_FLAGS) +def test_the_dense_localisation_flags_are_gone(tmp_path, capsys, flag): + """The dense line takes its home values, BRMAs, private rents and + fuel-relief flags from the spine (#931), so a script that still passes a + retired localisation flag is refused, neither ignored nor read as an + abbreviation of a live flag.""" + + with pytest.raises(SystemExit): + arguments(tmp_path, flag, "none") + assert f"unrecognized arguments: {flag} none" in capsys.readouterr().err + + def test_a_dense_build_that_must_send_telemetry_stops_before_any_input_without_an_emitter( tmp_path, monkeypatch ): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_contract.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_contract.py index af288d6d8c..997c1d5b81 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_contract.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_contract.py @@ -139,6 +139,75 @@ def test_complete_dense_and_size_declarations_fit_existing_export_limits(househo ) +#: The dense line's localisation nodes, retired with their flags (#931): the +#: dense line takes home values, BRMAs, private rents and fuel-relief flags +#: from the spine. +_RETIRED_DENSE_NODES = ( + "uk.full.geography.brma", + "uk.full.geography.rfdr", + "uk.full.private_rent", + "uk.full.property.value", + "uk.full.property.band", +) + + +def test_the_dense_line_composes_and_rosters_no_localisation_node(): + """The atomic dense graph composes none of the retired nodes and neither + declares nor reads their resources, and the roster names none of them.""" + from microcosm.build.uk_runtime import ( + brma_area_layer, + private_rent_localisation, + property_localisation, + rfdr_area_layer, + ) + from microcosm.build.uk_runtime.atomic_area_support import ( + uk_atomic_assignment_definition, + ) + from test_support.microcosm_build.uk_atomic_support_fixtures import ( + toy_support_payloads, + ) + + spine = add_uk_spine_gate_nodes( + uk_spine_graph(source_mode="split"), + spec=load_country_spec("uk"), + engine_identity="declaration-only", + ) + config = UKFullBuildConfig(calibration_year=2025) + full = uk_full_graph( + config, + spine=spine, + atomic_geography_definition=uk_atomic_assignment_definition( + toy_support_payloads(), seed=config.seed + ), + ) + graph = append_uk_full_gate_nodes( + full.graph, + calibration=full.calibration, + spine_stage_names=uk_spine_endpoint(spine).stage_names, + engine_identity="declaration-only", + review_date="2026-10-08", + ) + spine_nodes = {node.id for node in spine.nodes} + dense = [node for node in graph.nodes if node.id not in spine_nodes] + composed = {node.id for node in dense} + assert "uk.full.pool" in composed + assert not composed & set(_RETIRED_DENSE_NODES) + resources = { + property_localisation.QUANTILES_SOURCE, + property_localisation.BAND_MIX_SOURCE, + property_localisation.SCOTLAND_BAND_MIX_SOURCE, + brma_area_layer.BRMA_AREA_LAYER_SOURCE, + private_rent_localisation.QUANTILES_SOURCE, + private_rent_localisation.COUNTS_SOURCE, + rfdr_area_layer.RFDR_AREA_LAYER_SOURCE, + } + assert not {name for node in dense for name in node.sources} & resources + assert not {source.name for source in graph.sources} & resources + stages = uk_spine_stage_ids() + for node_id in _RETIRED_DENSE_NODES: + assert uk_operation_group(node_id, spine_stages=stages) == "other" + + def test_old_checkpoint_has_an_explicit_missing_boundary(tmp_path): sidecar = tmp_path / "old.build.json" sidecar.write_text("{}") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_private_rent_localisation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_private_rent_localisation.py index c1a43d68d5..43ff0e673a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_private_rent_localisation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_private_rent_localisation.py @@ -11,22 +11,9 @@ from hypothesis import strategies as st from microcosm.build.uk_runtime import private_rent_localisation as prl -from microcosm.build.uk_runtime.atomic_area_support import ( - uk_atomic_assignment_definition, -) -from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig -from microcosm.graph import ContentStore, compile_graph, run_graph -from test_support.microcosm_build import uk_full_population_graph as population from test_support.microcosm_build.uk_atomic_support_fixtures import ( TOY_BRMA_RENT_LEVELS, toy_private_rent_resources, - toy_support_sources, - write_toy_brma_area_layer, - write_toy_private_rent_resources, - write_toy_supports, -) -from test_support.microcosm_build.uk_ladder_rowwise_clone import ( - toy_ladder as toy_ladder, ) W = prl.WEEKS_IN_YEAR @@ -374,207 +361,7 @@ def test_weeks_in_year_is_the_spines() -> None: # ---------------------------------------------------------------------------- -# The graph - - -def _graph(k: int, tmp_path, *, mode: str = "brma_list", full_scale: bool = True): - payloads, paths = write_toy_supports(tmp_path / "supports") - definition = uk_atomic_assignment_definition(payloads, seed=7) - graph, registry = population.graph_and_registry( - k, - geography_assignment="atomic", - definition=definition, - brma_localisation_mode="area", - private_rent_localisation_mode=mode, - full_scale=full_scale, - ) - return graph, registry, paths - - -@pytest.mark.parametrize("full_scale", [True, False]) -def test_the_rent_node_is_strict_exactly_at_full_scale(full_scale, tmp_path) -> None: - """microcosm#1305 re-review: the sampled-rung fallback is pinned to the rung.""" - - graph, _, _ = _graph(1, tmp_path, full_scale=full_scale) - params = graph.node("uk.full.private_rent").params - assert params["strict_cells"] is full_scale - - -@pytest.mark.parametrize( - "fractions,full_scale", - [((1.0, 1.0), True), ((0.5, 1.0), False), ((1.0, 0.5), False)], -) -def test_the_build_config_sets_full_scale_from_both_sample_fractions( - fractions, full_scale, monkeypatch -) -> None: - from microcosm.build.uk_runtime import graph_build - - seen = {} - - class StopBuildError(Exception): - pass - - def capture(*args, **kwargs): - seen.update(kwargs) - raise StopBuildError - - from microcosm.build.uk_runtime.graph import uk_spine_graph - - monkeypatch.setattr(graph_build, "append_uk_population_nodes", capture) - sample, source = fractions - with pytest.raises(StopBuildError): - graph_build.uk_full_graph( - UKFullBuildConfig( - calibration_year=2025, - sample_fraction=sample, - source_sample_fraction=source, - ), - spine=uk_spine_graph(source_mode="split"), - ) - assert seen["full_scale"] is full_scale - - -def _sources(tmp_path, ladder_path, paths) -> dict: - return { - "uk_ladder": ladder_path, - "fixture": ladder_path, - **toy_support_sources(paths), - **write_toy_brma_area_layer(tmp_path / "brma"), - **write_toy_private_rent_resources(tmp_path / "rent"), - } - - -@pytest.mark.parametrize("k", [1, 3]) -def test_private_rent_node_rewrites_the_renter_and_replays( - k, toy_ladder, tmp_path -) -> None: - _, ladder_path = toy_ladder - graph, registry, paths = _graph(k, tmp_path) - compiled = compile_graph(graph) - order = compiled.order - assert order.index("uk.full.geography.brma") < order.index("uk.full.private_rent") - assert order.index("uk.full.private_rent") < order.index("uk.full.geography_gate") - sources = _sources(tmp_path, ladder_path, paths) - store = ContentStore(tmp_path / "store") - run = run_graph(compiled, sources=sources, store=store, kernels=registry) - household = run.population("uk.full.expand").table("household") - renters = household[household["tenure_type"].astype(str) == "RENT_PRIVATELY"] - others = household[household["tenure_type"].astype(str) != "RENT_PRIVATELY"] - assert len(renters) == k and set(renters["brma"].astype(str)) == {"LOTHIAN"} - # One FRS renter in its cell: its rank u is its tie-break, and the region's - # scale sets the census-mix mean to its own spine rent of 9,000, so every - # clone reads 9,000 * (0.5 + u) off the linear toy list: one rent for all - # clones, and not the spine's. - assert renters["rent"].nunique() == 1 - rent = float(renters["rent"].iloc[0]) - assert 4_500.0 < rent < 13_500.0 - assert rent != pytest.approx(9_000.0) - assert set(others["rent"]) == {0.0} - receipt = run.nodes["uk.full.private_rent"].receipt - assert receipt["weights"] == "unchanged" - assert receipt["regions_scaled"] == 1 - _, fresh, _ = _graph(k, tmp_path / "fresh") - replay = run_graph( - compiled, sources=sources, store=store, kernels=fresh, resume="require" - ) - assert all(node.hit for node in replay.nodes.values()) - - -def test_london_clones_read_their_own_areas_list( - toy_ladder, tmp_path, monkeypatch -) -> None: - original = population.source_frame - - def london_renter(): - frame = original() - household = frame.table("household") - household.loc[0, "tenure_type"] = "RENT_PRIVATELY" - household.loc[0, "rent"] = 20_000.0 - household.loc[0, "num_bedrooms"] = 2 - return frame - - monkeypatch.setattr(population, "source_frame", london_renter) - _, ladder_path = toy_ladder - graph, registry, paths = _graph(6, tmp_path) - run = run_graph( - compile_graph(graph), - sources=_sources(tmp_path, ladder_path, paths), - store=ContentStore(tmp_path / "store"), - kernels=registry, - ) - household = run.population("uk.full.expand").table("household") - london = household[household["region"].astype(str) == "LONDON"] - # Six clones, placed in both toy London BRMAs (two and four at this seed): - # each BRMA's clones read one rent, in the ratio of the two lists. - assert len(london) == 6 - by_brma = london.groupby(london["brma"].astype(str))["rent"].agg(["min", "max"]) - assert set(by_brma.index) == {"CENTRAL_LONDON", "INNER_EAST_LONDON"} - assert (by_brma["min"] == by_brma["max"]).all() - assert not np.isclose(london["rent"].astype(float), 20_000.0).any() - ratio = ( - by_brma.loc["CENTRAL_LONDON", "min"] / by_brma.loc["INNER_EAST_LONDON", "min"] - ) - assert ratio == pytest.approx( - TOY_BRMA_RENT_LEVELS["CENTRAL_LONDON"] - / TOY_BRMA_RENT_LEVELS["INNER_EAST_LONDON"] - ) - - -@pytest.mark.parametrize("full_scale", [True, False]) -def test_the_rent_kernel_reads_its_strictness_off_the_node( - full_scale, toy_ladder, tmp_path, monkeypatch -) -> None: - """The third review of #1305: the kernel, not just the node, honours it. - - Hard-coding the kernel's strictness passed every test; the kernel must - hand ``localise_rents`` the node's own ``strict_cells``. - """ - seen: list[bool] = [] - original = prl.localise_rents - - def recording(**kwargs): - seen.append(kwargs["strict"]) - return original(**kwargs) - - monkeypatch.setattr(prl, "localise_rents", recording) - _, ladder_path = toy_ladder - graph, registry, paths = _graph(1, tmp_path, full_scale=full_scale) - run_graph( - compile_graph(graph), - sources=_sources(tmp_path, ladder_path, paths), - store=ContentStore(tmp_path / "store"), - kernels=registry, - ) - assert seen == [full_scale] - - -def test_none_mode_adds_no_private_rent_node(tmp_path) -> None: - graph, _, _ = _graph(1, tmp_path, mode="none") - assert "uk.full.private_rent" not in compile_graph(graph).order - - -def test_private_rents_need_area_brmas() -> None: - with pytest.raises(ValueError, match="need the area BRMAs"): - population.graph_and_registry( - 1, - geography_assignment="legacy", - private_rent_localisation_mode="brma_list", - ) - - -def test_config_follows_the_brma_mode() -> None: - assert ( - UKFullBuildConfig(calibration_year=2025).private_rent_localisation - == "brma_list" - ) - no_brma = UKFullBuildConfig(calibration_year=2025, brma_localisation="none") - assert no_brma.private_rent_localisation == "none" - with pytest.raises(ValueError, match="need the area BRMAs"): - UKFullBuildConfig( - calibration_year=2025, - brma_localisation="none", - private_rent_localisation="brma_list", - ) +# Sampled rungs def _regional_rows(*, wales_frs: int, wales_rent: float = 9_000.0) -> dict: @@ -671,43 +458,3 @@ def test_a_sampled_region_with_a_few_frs_renters_ranks_on_great_britain(): ) == frozenset() ) - - -def test_the_size_identity_moves_with_the_rent_pins(): - """microcosm#1305 review: a size checkpoint binds the rent resources it ran on.""" - from argparse import Namespace - - from microcosm.build.uk_runtime.rowwise_posture import UK_ROWWISE_DENSE_POSTURE - from microcosm.build.uk_runtime.size_checkpoint import uk_size_checkpoint_identity - - def identity(quantiles_sha256: str) -> dict: - args = Namespace( - _posture=UK_ROWWISE_DENSE_POSTURE, - ledger_facts_sha256="f", - ledger_manifest_sha256="m", - seed=7, - selection_seed=None, - n_clones=2, - dataset_households=2, - epochs=2, - learning_rate=0.02, - sample_fraction=1.0, - sample_seed=1, - source_lineage_modulus=None, - target_weight_rule="grain_equal", - engine_blocks=1, - measure_exclusions=None, - private_rent_localisation="brma_list", - ) - pins = { - "dataset": {"sha256": "d"}, - "ladder": {"sha256": "l"}, - prl.QUANTILES_SOURCE: {"sha256": quantiles_sha256}, - prl.COUNTS_SOURCE: {"sha256": "c"}, - } - return uk_size_checkpoint_identity(args, pins=pins, source_year=2024) - - assert identity("q1") != identity("q2") - assert identity("q1")["private_rent_localisation"]["resource_pins"][ - prl.QUANTILES_SOURCE - ] == {"sha256": "q1"} diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_localisation.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_localisation.py index 36c7e574bf..627de4070e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_property_localisation.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_property_localisation.py @@ -9,25 +9,12 @@ import pytest from microcosm.build.uk_runtime import property_localisation as pl -from microcosm.build.uk_runtime.atomic_area_support import ( - uk_atomic_assignment_definition, -) from microcosm.build.uk_runtime.household_lineage import ( uk_household_clone_paths, without_geographic_branch, ) -from microcosm.graph import ContentStore, compile_graph, run_graph from test_support.microcosm_build.uk_atomic_support_fixtures import ( toy_property_resources, - toy_support_sources, - write_toy_property_resources, - write_toy_supports, -) -from test_support.microcosm_build.uk_full_population_graph import ( - graph_and_registry, -) -from test_support.microcosm_build.uk_ladder_rowwise_clone import ( - toy_ladder as toy_ladder, ) STREAM = ("sha256-u53-v1", pl.PROPERTY_STREAM_EXPERIMENT, 0, 7) @@ -406,79 +393,3 @@ def test_spine_keys_drop_the_geographic_branch_only() -> None: assert spine[0] == spine[1] == () keys = pl.spine_household_keys(household, source="frs", source_vintage="v") assert keys[0] == keys[1] != keys[2] - - -# ---------------------------------------------------------------------------- -# Graph - - -@pytest.mark.parametrize("k", [1, 3]) -def test_property_nodes_rewrite_values_and_bands_per_clone_and_replay( - k, toy_ladder, tmp_path -) -> None: - _, ladder_path = toy_ladder - payloads, paths = write_toy_supports(tmp_path / "supports") - resource_paths = write_toy_property_resources(tmp_path / "property") - definition = uk_atomic_assignment_definition(payloads, seed=7) - graph, registry = graph_and_registry( - k, - geography_assignment="atomic", - definition=definition, - property_localisation_mode="price_paid", - ) - compiled = compile_graph(graph) - order = compiled.order - assert order.index("uk.full.geography.derive") < order.index( - "uk.full.property.value" - ) - assert order.index("uk.full.property.value") < order.index("uk.full.property.band") - assert order.index("uk.full.property.band") < order.index("uk.full.geography_gate") - sources = { - "uk_ladder": ladder_path, - "fixture": ladder_path, - **toy_support_sources(paths), - **resource_paths, - } - store = ContentStore(tmp_path / "store") - first = run_graph(compiled, sources=sources, store=store, kernels=registry) - household = first.population("uk.full.expand").table("household") - value_receipt = first.nodes["uk.full.property.value"].receipt - band_receipt = first.nodes["uk.full.property.band"].receipt - england_wales = household["constituency_code"].astype(str).str[0].isin(["E", "W"]) - owners = household["tenure_type"].isin( - ["OWNED_OUTRIGHT", "OWNED_WITH_MORTGAGE"] - ) & (household["main_residence_value"] > 0) - # Every England-and-Wales owner clone reads its value off the toy knots. - rewritten = household[england_wales & owners] - assert len(rewritten) == 2 * k and value_receipt["rows_rewritten"] == 2 * k - assert (rewritten["main_residence_value"] >= 100_000).all() - assert (rewritten["main_residence_value"] <= 2_000_000).all() - # Clones of one spine household share one uniform, so one value. - per_source = rewritten.groupby("source_household_id")[ - "main_residence_value" - ].nunique() - assert (per_source == 1).all() - # Scotland and Northern Ireland keep the spine's values and bands. - untouched = household[~england_wales] - assert set(untouched["main_residence_value"]) <= {0.0} - assert band_receipt["rows_untouched_outside_england_wales"] == len(untouched) - assert band_receipt["monotone_violations"] == 0 - assert set(household.loc[england_wales, "council_tax_band"]) <= set("ABCDEFGH") - assert value_receipt["weights"] == "unchanged" - _, fresh = graph_and_registry( - k, - geography_assignment="atomic", - definition=definition, - property_localisation_mode="price_paid", - ) - replay = run_graph( - compiled, sources=sources, store=store, kernels=fresh, resume="require" - ) - assert all(receipt.hit for receipt in replay.nodes.values()) - - -def test_property_localisation_needs_the_atomic_assignment() -> None: - with pytest.raises(ValueError, match="atomic geography assignment"): - graph_and_registry( - 1, geography_assignment="legacy", property_localisation_mode="price_paid" - ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_regional_home_values.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_regional_home_values.py index 3f0c2756c8..c457f2eb60 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_regional_home_values.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_regional_home_values.py @@ -13,14 +13,6 @@ from microcosm.build.country_spec import load_country_spec from microcosm.build.uk_runtime import property_localisation from microcosm.build.uk_runtime import regional_home_values as rhv -from microcosm.build.uk_runtime.atomic_area_support import ( - uk_atomic_assignment_definition, -) -from microcosm.build.uk_runtime.graph_kernels import UKClaimKernel -from microcosm.build.uk_runtime.graph_population import ( - append_uk_population_nodes, - register_uk_population_kernels, -) from microcosm.build.uk_runtime.household_lineage import HIGH_VALUE_SUPPORT_COPY_COLUMN from microcosm.build.uk_runtime.national_frame import ( UK_RELEASE_EXPORT_DROPPED_COLUMNS, @@ -28,30 +20,7 @@ ) from microcosm.build.uk_runtime.receipt_counts import support_band from microcosm.frame import Frame, WeightKind -from microcosm.graph import ( - ContentStore, - Graph, - KernelRegistry, - KernelResult, - SourceRef, - compile_graph, - run_graph, -) -from test_support.microcosm_build.uk_atomic_support_fixtures import ( - toy_support_sources, - write_toy_property_resources, - write_toy_supports, -) from test_support.microcosm_build.uk_clone_first_frames import clone_first_frame -from test_support.microcosm_build.uk_full_population_graph import ( - SOURCE_VINTAGE, - Source, - source_frame, - source_node, -) -from test_support.microcosm_build.uk_ladder_rowwise_clone import ( - toy_ladder as toy_ladder, -) _REGIONS = ("LONDON", "NORTH_EAST", "WALES", "SCOTLAND", "NORTHERN_IRELAND") @@ -128,7 +97,7 @@ def test_england_and_wales_owners_take_their_regions_distribution() -> None: assert not np.allclose( after["main_residence_value"].to_numpy()[target], value[target] ) - # The replaced value is kept for the dense node to rank on. + # The replaced value is kept in the rank-basis column. np.testing.assert_array_equal(after[rhv.RANK_BASIS_COLUMN].to_numpy(), value) # Wealth moves by the value's change. np.testing.assert_allclose( @@ -175,105 +144,6 @@ def test_the_wales_map_uses_the_knots_country_code() -> None: assert len(rhv.FRS_REGION_TO_KNOTS_REGION) == 10 -class _FrameSource(Source): - """The toy source kernel, returning a given frame.""" - - def __init__(self, frame) -> None: - self._frame = frame - - def run(self, context): - return KernelResult(frame=self._frame) - - -def _dense_frame(*, rank_basis=None, value=None): - """The toy dense source with two London owners, so a value can flip a rank.""" - - base = source_frame() - tables = {entity: base.table(entity).copy() for entity in base.entities} - household = tables["household"] - household["region"] = pd.array( - ["LONDON", "LONDON", "SCOTLAND", "NORTHERN_IRELAND"], dtype="string" - ) - basis = np.array([400_000.0, 250_000.0, 0.0, 0.0]) - household["main_residence_value"] = basis - if rank_basis is not None: - household[property_localisation.RANK_BASIS_COLUMN] = rank_basis - household["main_residence_value"] = value - household["property_wealth"] = household["property_wealth"] + (value - basis) - return Frame( - tables, - base.schema, - {"household": base.weights_for("household")}, - base.strata, - mass_log=base.mass_log, - metadata=base.metadata, - ) - - -def _dense_households(frame, directory, ladder_path): - payloads, paths = write_toy_supports(directory / "supports") - resource_paths = write_toy_property_resources(directory / "property") - graph = append_uk_population_nodes( - Graph("uk", (SourceRef("fixture", "raw-bytes-v1"),), (source_node(frame),)), - population="source", - time_period="2023", - weight_kind="importance", - n_clones=1, - seed=7, - source_year=2023, - geography_assignment="atomic", - atomic_geography_definition=uk_atomic_assignment_definition(payloads, seed=7), - source_vintage=SOURCE_VINTAGE, - property_localisation_mode="price_paid", - ) - registry = KernelRegistry() - registry.register(_FrameSource(frame)) - registry.register(UKClaimKernel()) - register_uk_population_kernels(registry) - run = run_graph( - compile_graph(graph), - sources={ - "uk_ladder": ladder_path, - "fixture": ladder_path, - **toy_support_sources(paths), - **resource_paths, - }, - store=ContentStore(directory / "store"), - kernels=registry, - ) - household = run.population("uk.full.expand").table("household") - return household[["household_id", "main_residence_value", "property_wealth"]] - - -def test_the_dense_line_does_not_move_with_the_regional_values( - toy_ladder, tmp_path -) -> None: - _, ladder_path = toy_ladder - basis = np.array([400_000.0, 250_000.0, 0.0, 0.0]) - # The regional stage swaps the two London owners' order. - regional = np.array([200_000.0, 500_000.0, 0.0, 0.0]) - before = _dense_households(_dense_frame(), tmp_path / "before", ladder_path) - after = _dense_households( - _dense_frame(rank_basis=basis, value=regional), tmp_path / "after", ladder_path - ) - # The value node receives the replaced value through its slice, so the - # dense line's values and wealth are those it had before the stage. - pd.testing.assert_frame_equal( - before.reset_index(drop=True), after.reset_index(drop=True) - ) - # Ranking on the regional values instead would move them: the test is - # sensitive to the basis the node reads. - moved = _dense_households( - _dense_frame(rank_basis=regional, value=regional), - tmp_path / "moved", - ladder_path, - ) - assert not np.allclose( - before["main_residence_value"].to_numpy(), - moved["main_residence_value"].to_numpy(), - ) - - def test_the_rank_basis_refuses_a_declaration_mismatch() -> None: household = pd.DataFrame({"main_residence_value": [1.0, 2.0]}) np.testing.assert_array_equal( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rfdr_area_graph.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rfdr_area_graph.py deleted file mode 100644 index d9ebf717d1..0000000000 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rfdr_area_graph.py +++ /dev/null @@ -1,191 +0,0 @@ -"""The post-geography Rural Fuel Duty Relief node in the UK full-build graph (#1418).""" - -from __future__ import annotations - -import functools -import json - -import pytest - -from microcosm.build.uk_runtime.atomic_area_support import ( - uk_atomic_assignment_definition, -) -from microcosm.build.uk_runtime.graph_build import UKFullBuildConfig -from microcosm.build.uk_runtime.graph_population import _rfdr_area_nodes -from microcosm.build.uk_runtime.rfdr_area_layer import ( - RFDR_AREA_LAYER_SOURCE, - RFDR_COLUMN, -) -from microcosm.graph import ContentStore, compile_graph, run_graph -from microcosm.graph.errors import NodeRejectedError -from test_support.microcosm_build.uk_atomic_support_fixtures import ( - TOY_RFDR_AREAS, - toy_rfdr_area_layer, - toy_support_sources, - write_toy_rfdr_area_layer, - write_toy_supports, -) -from test_support.microcosm_build.uk_full_population_graph import ( - graph_and_registry, -) -from test_support.microcosm_build.uk_ladder_rowwise_clone import ( - toy_ladder as toy_ladder, -) - - -def _atomic(k: int, tmp_path, mode: str = "area", *, full_scale: bool = True): - payloads, paths = write_toy_supports(tmp_path / "supports") - definition = uk_atomic_assignment_definition(payloads, seed=7) - graph, registry = graph_and_registry( - k, - geography_assignment="atomic", - definition=definition, - rfdr_localisation_mode=mode, - full_scale=full_scale, - ) - return graph, registry, paths - - -@pytest.mark.parametrize("k", [1, 3]) -def test_rfdr_node_flags_each_clone_by_its_area_and_replays( - k, toy_ladder, tmp_path -) -> None: - _, ladder_path = toy_ladder - graph, registry, paths = _atomic(k, tmp_path) - compiled = compile_graph(graph) - order = compiled.order - # The node reads the assigned area and precedes the geography gates. - assert order.index("uk.full.geography.assign") < order.index( - "uk.full.geography.rfdr" - ) - assert order.index("uk.full.geography.rfdr") < order.index("uk.full.geography.gate") - sources = { - "uk_ladder": ladder_path, - "fixture": ladder_path, - **toy_support_sources(paths), - **write_toy_rfdr_area_layer(tmp_path / "rfdr"), - } - store = ContentStore(tmp_path / "store") - first = run_graph(compiled, sources=sources, store=store, kernels=registry) - household = first.population("uk.full.expand").table("household") - flagged = {code for codes in TOY_RFDR_AREAS.values() for code in codes} - expected = household["atomic_area_code"].astype(str).isin(flagged) - assert expected.any() and not expected.all() - assert household[RFDR_COLUMN].astype(bool).tolist() == expected.tolist() - # Northern Ireland has no area in the scheme. - ni = household["atomic_area_system"].astype(str) == "uk_ni_data_zone_2021" - assert not household.loc[ni, RFDR_COLUMN].astype(bool).any() - receipt = first.nodes["uk.full.geography.rfdr"].receipt - assert set(receipt["rows_by_system"]) == { - "uk_ew_output_area_2021", - "uk_ni_data_zone_2021", - "uk_scotland_output_area_2022", - } - _, fresh, _ = _atomic(k, tmp_path / "fresh") - replay = run_graph( - compiled, sources=sources, store=store, kernels=fresh, resume="require" - ) - assert all(node.hit for node in replay.nodes.values()) - - -def test_none_mode_adds_no_rfdr_node(tmp_path) -> None: - graph, _, _ = _atomic(1, tmp_path, mode="none") - assert "uk.full.geography.rfdr" not in compile_graph(graph).order - assert "uk_rfdr_area_layer" not in {source.name for source in graph.sources} - - -def test_area_rfdr_flags_need_the_atomic_assignment() -> None: - with pytest.raises(ValueError, match="atomic geography assignment"): - graph_and_registry( - 1, geography_assignment="legacy", rfdr_localisation_mode="area" - ) - with pytest.raises(ValueError, match="RFDR localisation must be one of"): - graph_and_registry(1, rfdr_localisation_mode="region") - - -def test_the_node_refuses_a_pool_without_its_columns() -> None: - cells = {("household", "atomic_area_system"): "string"} - with pytest.raises(ValueError, match="'atomic_area_code'"): - _rfdr_area_nodes(cells, common={"time_period": "2023"}) - - -def test_only_a_full_scale_build_refuses_a_system_with_none_flagged() -> None: - # Vahid's review and the independent review of microcosm#1420: a sampled - # rung can miss the 41 English and Welsh output areas by chance. - cells = { - ("household", "atomic_area_system"): "string", - ("household", "atomic_area_code"): "string", - } - ((full,), _) = _rfdr_area_nodes(dict(cells), common={"time_period": "2023"}) - ((sampled,), _) = _rfdr_area_nodes( - dict(cells), common={"time_period": "2023"}, strict=False - ) - assert full.params["strict_flags"] is True - assert sampled.params["strict_flags"] is False - - -def test_config_follows_the_geography_assignment() -> None: - assert UKFullBuildConfig(calibration_year=2025).rfdr_localisation == "area" - legacy = UKFullBuildConfig( - calibration_year=2025, - geography_assignment="legacy", - property_localisation="none", - ) - assert legacy.rfdr_localisation == "none" - with pytest.raises(ValueError, match="atomic geography assignment"): - UKFullBuildConfig( - calibration_year=2025, - geography_assignment="legacy", - property_localisation="none", - rfdr_localisation="area", - ) - - -@pytest.mark.parametrize("full_scale", [True, False]) -def test_the_rfdr_node_is_strict_exactly_at_full_scale(full_scale, tmp_path) -> None: - """The re-review of microcosm#1420, C: the graph sets the switch from the - build's own scale, not only a direct call.""" - - graph, _, _ = _atomic(1, tmp_path, full_scale=full_scale) - assert graph.node("uk.full.geography.rfdr").params["strict_flags"] is full_scale - - -@pytest.mark.parametrize("full_scale", [True, False]) -def test_the_rfdr_kernel_refuses_an_unflagged_system_only_at_full_scale( - full_scale, toy_ladder, tmp_path -) -> None: - """The re-review of microcosm#1420, C: the kernel honours the node's - switch. England and Wales's flagged area is one no household is placed - in: a full-scale run refuses, a sampled rung flags none there and runs.""" - - _, ladder_path = toy_ladder - graph, registry, paths = _atomic(1, tmp_path, full_scale=full_scale) - layer = toy_rfdr_area_layer() - layer["areas"]["uk_ew_output_area_2021"] = ["E00099999"] - layer_path = tmp_path / "unplaced-rfdr-area-layer.json" - layer_path.write_text(json.dumps(layer), encoding="utf-8") - sources = { - "uk_ladder": ladder_path, - "fixture": ladder_path, - **toy_support_sources(paths), - RFDR_AREA_LAYER_SOURCE: layer_path, - } - run = functools.partial( - run_graph, - compile_graph(graph), - sources=sources, - store=ContentStore(tmp_path / "store"), - kernels=registry, - ) - if full_scale: - with pytest.raises( - NodeRejectedError, match="flags none of the 2 uk_ew_output_area_2021" - ): - run() - return - household = run().population("uk.full.expand").table("household") - england_wales = ( - household["atomic_area_system"].astype(str) == "uk_ew_output_area_2021" - ) - assert england_wales.any() - assert not household.loc[england_wales, RFDR_COLUMN].astype(bool).any() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py index 91bc1f2ced..1379635762 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_rowwise_candidate.py @@ -94,15 +94,6 @@ def test_graph_driver_dry_run_prints_the_operation_inventory( "2", "--seed", "7", - # microcosm#1240: the fixture frame carries no housing columns, so the - # dry run keeps the spine's values rather than planning the value and - # band nodes. - "--property-localisation", - "none", - # microcosm#1262: nor a BRMA column, so the dry run keeps the spine's - # regional draw rather than planning the area-BRMA node. - "--brma-localisation", - "none", "--dry-run", "--no-staging", ] diff --git a/test_support/microcosm_build/uk_atomic_support_fixtures.py b/test_support/microcosm_build/uk_atomic_support_fixtures.py index 0d6761f002..fe44db9a1c 100644 --- a/test_support/microcosm_build/uk_atomic_support_fixtures.py +++ b/test_support/microcosm_build/uk_atomic_support_fixtures.py @@ -11,7 +11,6 @@ from __future__ import annotations -import json from pathlib import Path import numpy as np @@ -221,116 +220,6 @@ def toy_property_resources() -> dict[str, dict]: } -def write_toy_property_resources(directory: Path) -> dict[str, Path]: - """Write the toy knots and band mixes as graph sources named by the nodes.""" - from microcosm.build.uk_runtime.property_localisation import ( - BAND_MIX_SOURCE, - QUANTILES_SOURCE, - SCOTLAND_BAND_MIX_SOURCE, - ) - - directory.mkdir(parents=True, exist_ok=True) - resources = toy_property_resources() - paths = { - QUANTILES_SOURCE: directory / "toy-knots.json", - BAND_MIX_SOURCE: directory / "toy-band-mix.json", - SCOTLAND_BAND_MIX_SOURCE: directory / "toy-scotland-band-mix.json", - } - paths[SCOTLAND_BAND_MIX_SOURCE].write_text( - json.dumps(resources["scotland_band_mix"]), encoding="utf-8" - ) - paths[QUANTILES_SOURCE].write_text(json.dumps(resources["knots"]), encoding="utf-8") - paths[BAND_MIX_SOURCE].write_text( - json.dumps(resources["band_mix"]), encoding="utf-8" - ) - return paths - - -#: The BRMA the toy layer gives each toy area (microcosm#1262); every other -#: England-and-Wales filler area sits in MAIDSTONE. -TOY_AREA_BRMAS = { - "E00000001": "CENTRAL_LONDON", - "E00000002": "INNER_EAST_LONDON", - "W00000001": "CARDIFF", - "S00000001": "LOTHIAN", -} - - -def toy_brma_area_layer() -> dict: - """The microcosm#1262 area-to-BRMA layer over the toy supports.""" - from microcosm.build.uk_runtime import brma_area_layer as bal - - regions = {code: name for name, code in FRS_REGION_TO_REGION_CODE.items()} - frames = toy_support_frames() - layout: dict[str, dict] = {"ew_lsoa21": {}, "scotland_oa22": {}} - rosters: dict[str, list[str]] = {"ew_lsoa21": [], "scotland_oa22": []} - for grain, system, key in ( - ("ew_lsoa21", EW, "lsoa_code"), - ("scotland_oa22", SCOT, "oa_code"), - ): - for row in frames[system]: - code = str(row[key]) - brma = TOY_AREA_BRMAS.get(code, "MAIDSTONE") - region = regions[str(row["region_code"])] - layout[grain].setdefault(region, {}).setdefault(brma, []).append(code) - rosters[grain].append(code) - return { - "schema_version": bal.SCHEMA_VERSION, - "kind": bal.KIND, - "method": bal.METHOD_VERSION, - "grains": { - grain: { - "area_system": bal.GRAINS[grain].system, - "key": bal.GRAINS[grain].key, - "areas": len(codes), - "roster_sha256": bal.roster_sha256(codes), - } - for grain, codes in rosters.items() - }, - "scotland_brma_codes": {"LOTHIAN": "S33000011"}, - "brmas": layout, - } - - -def write_toy_brma_area_layer(directory: Path) -> dict[str, Path]: - """Write the toy layer as the graph source the area-BRMA node reads.""" - from microcosm.build.uk_runtime.brma_area_layer import BRMA_AREA_LAYER_SOURCE - - directory.mkdir(parents=True, exist_ok=True) - path = directory / "toy-brma-area-layer.json" - path.write_text(json.dumps(toy_brma_area_layer()), encoding="utf-8") - return {BRMA_AREA_LAYER_SOURCE: path} - - -#: The toy output areas in the Rural Fuel Duty Relief scheme (microcosm#1418). -TOY_RFDR_AREAS = { - "uk_ew_output_area_2021": ("E00000002",), - "uk_scotland_output_area_2022": ("S00000001",), -} - - -def toy_rfdr_area_layer() -> dict: - """The microcosm#1418 RFDR area layer over the toy supports.""" - from microcosm.build.uk_runtime import rfdr_area_layer as ral - - return { - "schema_version": ral.SCHEMA_VERSION, - "kind": ral.KIND, - "method": ral.METHOD_VERSION, - "areas": {system: list(codes) for system, codes in TOY_RFDR_AREAS.items()}, - } - - -def write_toy_rfdr_area_layer(directory: Path) -> dict[str, Path]: - """Write the toy layer as the graph source the area-RFDR node reads.""" - from microcosm.build.uk_runtime.rfdr_area_layer import RFDR_AREA_LAYER_SOURCE - - directory.mkdir(parents=True, exist_ok=True) - path = directory / "toy-rfdr-area-layer.json" - path.write_text(json.dumps(toy_rfdr_area_layer()), encoding="utf-8") - return {RFDR_AREA_LAYER_SOURCE: path} - - #: Toy weekly rent levels by BRMA; each category scales its BRMA's level. TOY_BRMA_RENT_LEVELS = { "CENTRAL_LONDON": 400.0, @@ -393,24 +282,4 @@ def toy_private_rent_resources() -> dict[str, dict]: return {"quantiles": quantiles, "counts": counts} -def write_toy_private_rent_resources(directory: Path) -> dict[str, Path]: - """Write the toy rent quantiles and counts as the rent node's graph sources.""" - from microcosm.build.uk_runtime.private_rent_localisation import ( - COUNTS_SOURCE, - QUANTILES_SOURCE, - ) - - directory.mkdir(parents=True, exist_ok=True) - resources = toy_private_rent_resources() - paths = { - QUANTILES_SOURCE: directory / "toy-brma-rent-quantiles.json", - COUNTS_SOURCE: directory / "toy-brma-private-renters.json", - } - paths[QUANTILES_SOURCE].write_text( - json.dumps(resources["quantiles"]), encoding="utf-8" - ) - paths[COUNTS_SOURCE].write_text(json.dumps(resources["counts"]), encoding="utf-8") - return paths - - __all__ = [name for name in globals() if not name.startswith("__")] diff --git a/test_support/microcosm_build/uk_full_population_graph.py b/test_support/microcosm_build/uk_full_population_graph.py index 4d1aa8a895..ae9f3ff5e2 100644 --- a/test_support/microcosm_build/uk_full_population_graph.py +++ b/test_support/microcosm_build/uk_full_population_graph.py @@ -61,9 +61,9 @@ def source_frame(): household["cgt_support_copy_index"] = np.zeros(len(household), dtype="int64") household["household_is_cgt_residential_clone"] = False household["cgt_residential_clone_index"] = np.zeros(len(household), dtype="int64") - # microcosm#1240: the housing columns the post-geography property nodes - # read and rewrite; one owner per region with a positive value, the - # Scottish household a renter, the Northern Irish one an owner at zero. + # The spine's housing columns: one owner per region with a positive + # value, the Scottish household a renter, the Northern Irish one an owner + # at zero. household["tenure_type"] = pd.array( ["OWNED_OUTRIGHT", "OWNED_WITH_MORTGAGE", "RENT_PRIVATELY", "OWNED_OUTRIGHT"][ : len(household) @@ -79,14 +79,12 @@ def source_frame(): household["council_tax_band"] = pd.array( ["D", "C", "B", "D"][: len(household)], dtype="string" ) - # microcosm#1262: the spine's regional BRMA draw, which the area-BRMA node - # replaces outside Northern Ireland. + # The spine's BRMA draw. household["brma"] = pd.array( ["MAIDSTONE", "MAIDSTONE", "MAIDSTONE", "BELFAST"][: len(household)], dtype="string", ) - # The Scottish household is the toy's one private renter; the private-rent - # node re-draws its rent from its area's toy list. + # The Scottish household is the toy's one private renter. household["rent"] = np.asarray( [0.0, 0.0, 9_000.0, 0.0][: len(household)], dtype="float64" ) @@ -149,11 +147,6 @@ def graph_and_registry( geography_assignment="legacy", definition=None, seed=7, - property_localisation_mode="none", - brma_localisation_mode="none", - private_rent_localisation_mode="none", - rfdr_localisation_mode="none", - full_scale=True, ): graph = append_uk_population_nodes( Graph( @@ -170,11 +163,6 @@ def graph_and_registry( geography_assignment=geography_assignment, atomic_geography_definition=definition, source_vintage=SOURCE_VINTAGE if geography_assignment == "atomic" else None, - property_localisation_mode=property_localisation_mode, - brma_localisation_mode=brma_localisation_mode, - private_rent_localisation_mode=private_rent_localisation_mode, - rfdr_localisation_mode=rfdr_localisation_mode, - full_scale=full_scale, ) registry = KernelRegistry() registry.register(Source()) diff --git a/test_support/microcosm_build/uk_full_target_graph.py b/test_support/microcosm_build/uk_full_target_graph.py index 8dc897be91..7603801105 100644 --- a/test_support/microcosm_build/uk_full_target_graph.py +++ b/test_support/microcosm_build/uk_full_target_graph.py @@ -50,10 +50,6 @@ ) from test_support.microcosm_build.uk_atomic_support_fixtures import ( toy_support_sources, - write_toy_brma_area_layer, - write_toy_private_rent_resources, - write_toy_property_resources, - write_toy_rfdr_area_layer, write_toy_supports, ) from test_support.microcosm_build.uk_full_calibration_graph import ( @@ -346,15 +342,7 @@ def build( "uk_ladder": ladder_path, "uk_ledger_facts": ladder_path, **( - { - **toy_support_sources(support_paths), - # microcosm#1240: the atomic assignment runs the value - # and band nodes, which read the toy knots and band mix. - **write_toy_property_resources(tmp_path / "property"), - **write_toy_brma_area_layer(tmp_path / "brma"), - **write_toy_private_rent_resources(tmp_path / "rent"), - **write_toy_rfdr_area_layer(tmp_path / "rfdr"), - } + toy_support_sources(support_paths) if geography_assignment == "atomic" else {} ), diff --git a/tools/uk_private_rent_pool.py b/tools/uk_private_rent_pool.py index 3ad52c03f8..b21b7932b2 100644 --- a/tools/uk_private_rent_pool.py +++ b/tools/uk_private_rent_pool.py @@ -9,6 +9,10 @@ to their spine household by the key the rent node ranks with (``property_localisation.spine_household_keys``). +Both nodes retired from the dense line in microcosm#931 group (c), which moves +BRMAs and private rents into the spine's own stages, so the tool reads only +dense runs built before then. + The table is licensed output. It stays in the run's audit directory, outside every git work tree, which the tool checks before it loads anything. Loading the pool frame takes about 15 GB of memory, so run this when no build is diff --git a/tools/uk_private_rent_receipt.py b/tools/uk_private_rent_receipt.py index 194a866d54..74a69b3e79 100644 --- a/tools/uk_private_rent_receipt.py +++ b/tools/uk_private_rent_receipt.py @@ -2,9 +2,12 @@ ``uk.full.geography.brma`` reads each placed Great Britain household's BRMA off its assigned area, and ``uk.full.private_rent`` redraws each private -renter's rent from that BRMA's list of rents. This tool measures one build, -and a control build when one is given, so that the PR's receipts and signed -entries rest on measured figures rather than on a gate: +renter's rent from that BRMA's list of rents. Both nodes retired from the +dense line in microcosm#931 group (c), which moves BRMAs and private rents +into the spine's own stages, so the tool reads only dense runs built before +then. It measures one build, and a control build when one is given, so that +the PR's receipts and signed entries rest on measured figures rather than on +a gate: - **Rent rows.** The bound ``ons.rent.private_rent`` (local authority) and ``ons.rent.private_rent_brma`` (Scottish BRMA) rows at the problem's design From 151ca6079e40891c8352d4d8733bc23fd46720f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:35:53 +0000 Subject: [PATCH 071/108] Build the dense line on the bound spine's clones, areas and sampling (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dense line no longer samples, expands, keys or places households of its own. Its targets, measures and problem run on the spine's version as it stands, its pool checkpoint keeps every row of that version, and its geographic clone count K, its sampling rung and seed and its geography binding are the spine's. It still refuses, before it pins or reads any input, every spine whose roster is not geography-first (spine_posture), so no dense build runs until the spine declares its area stages; the national line still refuses every clone-first spine. Why. María's ruling of 10 Nov (decision 4) holds the dense and national lines back until (c) re-bases them, and C-21 re-bases the dense line in three commits, refusal first: the roster refusal (commit 2), the four localisation families (commit 3) and this one, the last. K, sampling and the geography binding now come from the spine, so no dense node draws them a second time and the run records the spine's K and rung. - graph_population: uk.full.sample, uk.full.normalize, uk.full.expand (with uk.full.expand.owned), uk.full.identity, uk.full.locations, uk.full.geography_mapping, uk.full.geography.local_authority and uk.full.geography_gate go, with their eight kernels, the population-receipt, location-draw and geography-gate artifact types, append_uk_population_nodes, uk_pool_validation_inputs, GEOGRAPHY_ASSIGNMENTS and UK_GEOGRAPHY_IDENTITY_INPUTS. The UK registry stops registering the four shared atomic-geography kernels (NZ's transport registers its own), so the uk.full.geography.* support, assign, derive and gate nodes go too. uk.full.pool@1 keeps its id: a keep-all FILTER on uk.full.spine_checkpoint (--input-h5) or the spine's endpoint (--spine-request), with no typed edge, declared bitwise (platform-bitwise before, for the shared gate's edge) and hashing its own module and national_frame. The dense geography gate's ladder and installed-engine local-authority checks and the pool's refusal of a failed shared validation leave with their nodes; the spine's geography gates take them over once they are registered (commit 11), and until then the roster refusal holds every dense build. - graph_targets: append_uk_target_nodes takes population with no default. The measures lose the geography-gate edge and check, and take the spine's transferred-gate admission, which the sample node held, wherever the graph declares spine.gates.transferred; they check it before they read anything. - graph_build: UKFullBuildConfig loses sample_fraction, sample_seed, geography_assignment, source_vintage, constituency_vintage and source_lineage_modulus; n_clones defaults to 1, and effective_sample_fraction is source_sample_fraction, the spine's rung. uk_full_graph drops spine_weight_kind and atomic_geography_definition and refuses a composed spine whose geographic_expand node declares another K, naming both; build_uk_country_graph drops atomic_geography_definition. uk.full.bound_spine@1 also hashes geographic_clones. - calibration_run: _assert_spine_sidecar_binds_frame holds the sidecar's geographic_clone_count (one where a sidecar predates it) to the frame's, naming both; spine_provenance_from_sidecar adds K, geography, sampling and release_posture. - full_build_cli and rowwise_cli: prepare_full_build reads K, sampling, release posture and the geography binding from the authenticated sidecar, or from the spine request's --n-clones, --sample-fraction and --sample-seed, and sets the run's n_clones, sample_fraction and, for a sampled spine, sample_seed before the parameters, identity digest, size-checkpoint identity, staging rung and Logbook rungs record them. A spine whose sampling fraction is not a scale-ladder rung is refused, naming the rungs only. The attempt's build id is minted with the spine's rung, read ahead of preparation (unauthenticated, for the label) off the bound spine's sidecar or the request's --sample-fraction through the spine's own parser, and f100 where that read fails; preparation refuses an attempt whose build id carries another rung than the spine it binds, naming both rung tokens, and a refusal before the spine is bound records the build id's rung on its Logbook row. The request bindings gain a spine block, and geography is the spine's binding with K beside it. microcosm-build-uk no longer takes --geography-assignment, --atomic-support-ew, --atomic-support-scotland, --atomic-support-ni, --atomic-support-sha256-ew, --atomic-support-sha256-scotland, --atomic-support-sha256-ni or --source-lineage-modulus; the parser refuses each as an unrecognized argument on either role. The dense role refuses, by name and before any input is read, a --sample-fraction other than 1.0, any --sample-seed, --candidate-clone-counts with --input-h5, and --n-clones with --candidate-clone-counts. --n-clones only asserts the spine's K: the role no longer fills it from the doctrine, and preparation refuses one that differs, naming both, and on a spine request pointing at the request's own --n-clones. --candidate-clone-counts plans the spine request once per count, as its --n-clones. --expected-constituency-vintage is still taken and recorded, and its help now says no dense node checks the ladder against it. Release candidates: a sampled spine is refused at preparation, where its rung is read, on either route; the doctrine-K check and the support-pin requirement leave validation for the spine's K pin and binding check (commit 5); the ladder, Ledger and input pins, target rule, epochs, measure exclusions, local target levels and signing key are refused as before. - graph_terminal: the rowwise manifest reads sampling, identity.sampling, the rung and the release verdict's full-rung leg from the request's spine block, refusing a descriptor without it, and its ladder gate from the export descriptor's geography check, at phase post_calibration; the weight-kind chain names geographic_expand. - national_sampling: the dense family sampler (sample_uk_spine_frame, uk_spine_source_family_units) and its re-exports go; the root's sampler, the rung tokens and the default seed stay. - size_checkpoint: the identity loses source_lineage_modulus and the atomic geography block. - household_lineage: the reader's three lineage refusals open with the household table and its clone paths, not "Geography identity", and UK_OPTIONAL_LINEAGE_COLUMNS, which only the retired identity kernel read, goes. - Texts: rowwise_dataset's clone refusals point at the dense graph on a geography-first spine and promise no later group; tools/build_uk_rowwise_dataset.py says it serves spines without clone indices; the docstrings of geographic_identity, household_lineage (owed by commit 3), geographic_clones (which now names the national gates among its hashers) and diagnostics (whose support-family key comment names the lineage columns, not the identity kernel's inputs) stop naming the dense identity node or expansion as a writer or reader; tools/measure_uk_atomic_assignment.py refuses its graph route, and its command before it reads any input, with a pointer to the spine's atomic_area_assign stage; the Orrery roster loses its geography group and the retired nodes. Three older unreleased fragments that this change makes stale (the #932 wiring, the national sampler's retirement, commit 3's) each gain a clause saying what retires later. Defaults taken, each listed for María with its options: the configuration's K defaults to 1; a composed spine of another K is refused at composition, and a bound spine's K is read at preparation; a sidecar without geographic_clone_count reads as K = 1; the run's arguments are rewritten to the spine's K, fraction and seed; the spine binding's shape, with the manifest failing closed without it; --candidate-clone-counts replaces a request's own --n-clones, and --n-clones beside it is refused; the measures take the admission only where the graph declares the transferred gate; the pool is bitwise; the manifest's ladder gate is the descriptor's check at phase post_calibration; the weight-chain label is geographic_expand; the Orrery geography group goes; the identity node's tests retire with it and the lineage reader's own tests move to test_uk_household_lineage.py; the local area support gate's row, effective-sample-size and source floors are kept K-sensitive constants; the build id's rung is read ahead and held to the spine's at preparation; a sampling fraction off the ladder is refused; --expected-constituency-vintage stays a recorded assertion. Left for later: - The geographic_identity stage's notes in sources.yaml, and so source_stages.json and the H2 fixture, still name uk.full.identity: commit 8, which regenerates H2 with the geography declarations. - On a spine request the geography binding carries K alone: commit 8 composes the request route's block with the sidecar writer's helper, or commit 5 refuses --release-candidate with --spine-request; commit 5's binding check also derives the stand-in sidecar's geography values from the stand-in's own definition. - The spine's mapping-integrity, ladder and LA-engine-domain gates that take over the retired dense checks are declared release-blocking when commit 11 registers them. - --expected-constituency-vintage: commit 5's binding check, or a ruling (hold it to the ladder, retire it, or keep it recorded). - tools/measure_uk_atomic_assignment.py's manifest readers, past the refusal, still name retired nodes: commit 9 re-points the tool. - refuse_dense_clone_id_overflow keeps its name, though only the row-wise lane calls it, as do two row-wise driver test names; test_uk_national_graph.py's "group (c)" text goes with commit 12. - docs/uk-full-build-graph.md, docs/geography-assignment.md, docs/uk-atomic-household-lineage.md and the docstrings of tools/build_uk_brma_area_layer.py, tools/build_uk_rfdr_area_layer.py, tools/uk_private_rent_pool.py and tools/uk_private_rent_receipt.py: (f). - Until commit 5, validation holds no release candidate to the doctrine's K or the conditioned register; every dense build stops at the roster refusal first. - The PR body's sentence that dense multi-clone builds return "until (c)" is reworded at this commit's push. What moves. Keys and records only; no dataset value, and none at K = 1. No spine stage's code changes: household_lineage, geographic_clones and geographic_identity change in docstrings, comments and refusal texts (household_lineage also loses a constant nothing reads), and national_sampling loses only the dense sampler, which no stage calls. Through the engine slot, the H2 legacy oracle's final frame, stage records and stage evidence are identical at 8030471a7 and here. The dense line refuses every spine until commit 11, and a national build on a spine without clone indices computes the same dataset. - Kernels: the UK registries register 84 (96 before: the eight dense population kernels and the four shared atomic-geography ones go), and 64 change hash. 39 are of the spine registry's 58: uk.create@1, the 35 stages that hash household_lineage (geographic_identity, which also hashes its own module, among them), and cgt_support_split, was_wealth and was_lisa, which hash geographic_clones alone. 25 are uk.full kernels that hash an edited module: uk.full.target_compilation@1, uk.full.target_selection@1, uk.full.measures@1, uk.full.problem@1, uk.full.pool@1, uk.full.dense@1, uk.full.calibrated@1, the five size kernels (uk.full.size_checkpoint_import@1, uk.full.size_search@1, uk.full.size_draw@1, uk.full.size_refit@1 and uk.full.selected@1), uk.full.rotated-holdout@1, uk.full-gates.preflight@1, uk.full-gates@1, uk.full.certification-readiness@1, uk.full-export.prepare@1, uk.full-export.prepare-inspection@1, uk.full-export.package@1, uk.full-export.readback@1, uk.full.bound_spine@1, uk.full.national_targets@1, uk.full.national_problem@1, uk.full.national-gates@1 and uk.full.national-readback@1. uk.spine-gates@1, uk.claim@1, uk.identity@1 and 17 stages keep theirs. No kernel hashes full_build_cli or rowwise_cli, and diagnostics only the national gates and problem. - Node keys, on composed graphs with one stand-in content key per source: all 84 nodes of the declared spine with both gate batteries re-key (a spine build's root also hashes national_sampling through its sampler). The dense graph on that spine loses 13 nodes (uk.full.sample, uk.full.normalize, uk.full.expand, uk.full.expand.owned, uk.full.identity, uk.full.geography.support.0, .1 and .2, uk.full.geography.assign, uk.full.geography.derive, uk.full.geography.gate, uk.full.geography.local_authority and uk.full.geography_gate), and its other ten re-key, uk.full.measures also through its spine-gate parameters and uk.full.gates.calibrated through its terminal_code. On a stand-in bound spine the same 13 go and all eleven left re-key; the national graph's six nodes re-key. - No UK kernel is platform-bitwise now, so no dense node key carries the platform; on the declared spine 16 of the parent's 23 dense nodes did. - Of the gate phase codes only the full battery's terminal code moves. - Records: a dense run's input pins lose the three atomic-area supports, its parameters lose source_lineage_modulus, and its size-checkpoint identity loses both that and the geography block, so its input-pins digest, candidate identity digest and size-checkpoint identity change, and a size checkpoint cut before this change does not resume. The bound spine's provenance, which the national build record, the national calibration diagnostics' build block and the dense manifest carry, gains K, geography, sampling and release_posture. The rowwise manifest's sampling is the spine binding's (fraction, seed, sampled, rung token), and its ladder gate the descriptor's passed and failures. A dense build id on a sampled spine now carries the spine's rung token where it carried f100. - Unchanged: the NZ toy transport registry's 45 kernel hashes, the six US kernels and the US post-transfer graph's eight node keys, the spec engine's compiled am, be, nz, us and uk bundles and its kernel registry digest, and the packaged country specs. No declaration, generated file, gates.json or schema changes, and H2 is not regenerated. Tests (invented data): - Removed with the nodes they ran: test_uk_full_population_graph.py and test_uk_atomic_identity_graph.py. Their tests of live code move: the lineage reader's branch paths at one and two clones, residential arms and refusals to the new test_uk_household_lineage.py, which runs uk_household_clone_paths itself, and the row-wise lane's clone and source-lineage refusals, with their new texts, to test_uk_rowwise_dataset.py. - test_support/microcosm_build/uk_full_population_graph.py is a clone-first, geography-first spine stand-in: the toy households at K clones in the copy-id table's geographic_support field, keyed by household_draw_key and placed on the toy supports with the shared atomic operators, with a BRMA and a fuel-relief flag. The CLI, target-graph and row-wise stand-ins bind it. - New: each retired flag refused on both roles; the dense role's spine-bound refusals on both routes, --n-clones with --candidate-clone-counts among them; K, sampling and seed taken from the spine on both routes (a seven-clone spine without --n-clones records seven in the parameters, the identity digest, the configuration, the bindings and a written manifest; a sampled spine at a non-default seed gives the run that seed); a disagreeing --n-clones on both routes; a sidecar whose K is not its frame's; engine blocks held to the spine's K on both routes; --candidate-clone-counts planning a spine request once per count, with or without the request's own --n-clones; a release candidate on a sampled spine on both routes; a sampling fraction off the ladder; the build id's rung read ahead, carried by a sampled run's id and staging run, held to the spine's at preparation and recorded on an early refusal's Logbook row; a composed spine of another K; the measures' admission and its order; the pool on the spine's version and its keep-all mask and mass policy; the manifest's spine-bound sampling and ladder gate; the dense allow-list pin (what a geography-first spine adds to the export surface is in UK_ALLOWED_EXTRA_EXPORT_COLUMNS); the local area support gate's three floors in the K-sensitive constants table; the measurement tool's refusal, its command's included; geographic_identity's keys held to household_draw_key; the root's sampler in test_uk_national_sampling.py. - Reworded: the row-wise lane's id-rule reasons and refusal tests in test_uk_copy_id_table.py, and test_uk_graph.py's binding-table reasons, kernel census and the pool's hashed modules. Checks: through the engine slot, test_uk_full_target_graph, test_uk_full_graph_admission and test_uk_country_adapter (10 passed); the H2 legacy oracle identity of a fresh 8030471a7 snapshot against this tree (equal: 56 stages in the same order, the three tables identical, the weights bitwise equal, the mass log, metadata and stage records equal, and 48 stage-evidence digests and 56 evidence documents equal); the H2 fixture tool's --check (uk_spine regenerates byte-identically); and test_acceptance_h_parity.py (passed). In memory, five mutants of this change's binding (the request's K ignored, the identity digest taken before binding, the spine's seed dropped, the request route's --n-clones check skipped, the rung check dropped) each fail the preparation tests. Engine-free, a batch of the 158 files that import an edited module, test_uk_graph.py in full among them: 5,109 passed, 23 skipped and 2 failed, both environment-only: test_uk_cgt_projection's engine-unavailable test fails wherever the uk extra is installed, as here, and CI's engine-free job installs none; test_uk_orrery_reconstruction's Orrery check fails on this machine only. The coverage manifest's --check, ci_test_plan.py verify, ruff check and ruff format --check on the edited Python files, and git diff --check. An engine-free probe that reads code only compared this tree with its parent: every UK kernel's implementation hash and the edited modules it names, the gate phase codes, the node keys of the declared spine, of the dense graph on it and on a stand-in bound spine, and of the national graph on the stand-in, with one stand-in content key per source, the dense graph's platform-scoped keys under two invented platforms, and the NZ, US, spec-engine and packaged-spec identities above. Co-Authored-By: Claude Opus 5.5 --- ...ense-localisation-nodes-retired.removed.md | 2 +- ...-dense-on-the-clone-first-spine.changed.md | 18 + ...-retire-legacy-national-sampler.removed.md | 2 +- .../931-uk-atomic-geography-wiring.changed.md | 2 +- .../microcosm/build/uk_runtime/__init__.py | 6 - .../build/uk_runtime/calibration_run.py | 28 + .../build/uk_runtime/country_adapter.py | 9 +- .../microcosm/build/uk_runtime/diagnostics.py | 10 +- .../build/uk_runtime/full_build_cli.py | 398 +++++--- .../build/uk_runtime/geographic_clones.py | 13 +- .../build/uk_runtime/geographic_identity.py | 35 +- .../microcosm/build/uk_runtime/graph_build.py | 131 +-- .../build/uk_runtime/graph_population.py | 882 +----------------- .../build/uk_runtime/graph_targets.py | 51 +- .../build/uk_runtime/graph_terminal.py | 49 +- .../build/uk_runtime/household_lineage.py | 47 +- .../build/uk_runtime/national_sampling.py | 260 +----- .../build/uk_runtime/orrery_contract.py | 22 +- .../microcosm/build/uk_runtime/rowwise_cli.py | 155 ++- .../build/uk_runtime/rowwise_dataset.py | 68 +- .../build/uk_runtime/size_checkpoint.py | 19 +- .../engine/uk/test_uk_country_adapter.py | 18 +- .../engine/uk/test_uk_full_graph_admission.py | 25 +- .../engine/uk/test_uk_full_target_graph.py | 55 +- .../test_uk_atomic_assignment_measurement.py | 28 + .../uk/test_uk_atomic_identity_graph.py | 425 --------- .../engine_free/uk/test_uk_copy_id_table.py | 49 +- .../engine_free/uk/test_uk_full_build_cli.py | 369 ++++++-- .../uk/test_uk_full_build_preparation.py | 502 ++++++++-- .../uk/test_uk_full_population_graph.py | 418 --------- .../uk/test_uk_full_target_graph.py | 162 +++- .../uk/test_uk_geographic_identity.py | 84 +- .../tests/engine_free/uk/test_uk_graph.py | 110 +-- .../engine_free/uk/test_uk_graph_evidence.py | 103 +- .../engine_free/uk/test_uk_graph_terminal.py | 58 +- .../uk/test_uk_household_lineage.py | 264 ++++++ .../uk/test_uk_k_sensitive_constants.py | 61 +- .../uk/test_uk_national_sampling.py | 340 ++----- .../engine_free/uk/test_uk_orrery_contract.py | 49 +- .../uk/test_uk_orrery_reconstruction.py | 1 - .../uk/test_uk_rowwise_candidate.py | 78 +- .../engine_free/uk/test_uk_rowwise_dataset.py | 88 ++ .../tests/engine_free/uk/test_uk_size_l2.py | 1 - .../microcosm_build/uk_full_build_cli.py | 191 ++-- .../uk_full_population_graph.py | 216 ++++- .../microcosm_build/uk_full_target_graph.py | 41 +- .../microcosm_build/uk_rowwise_candidate.py | 40 +- tools/build_uk_rowwise_dataset.py | 6 + tools/measure_uk_atomic_assignment.py | 131 +-- 49 files changed, 2531 insertions(+), 3589 deletions(-) create mode 100644 changelog.d/931-dense-on-the-clone-first-spine.changed.md delete mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_atomic_identity_graph.py delete mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_full_population_graph.py create mode 100644 packages/microcosm-build/tests/engine_free/uk/test_uk_household_lineage.py diff --git a/changelog.d/931-dense-localisation-nodes-retired.removed.md b/changelog.d/931-dense-localisation-nodes-retired.removed.md index 86e6b3e6d4..c352a0bafa 100644 --- a/changelog.d/931-dense-localisation-nodes-retired.removed.md +++ b/changelog.d/931-dense-localisation-nodes-retired.removed.md @@ -1,6 +1,6 @@ Retire the dense line's own property, BRMA, private-rent and fuel-relief nodes, with their flags, configuration fields and release-candidate refusals (#931). The dense line takes home values, BRMAs, private rents and Rural Fuel Duty Relief flags from the spine, whose own stages set them in later commits of #931 group (c), and keeps the spine's council-tax bands. It already refuses, before it pins or reads any other input, a spine whose roster lacks those stages (`spine_posture`), so no dense build runs until they land. -- **Nodes and kernels.** `uk.full.property.value@1`, `uk.full.property.band@1`, `uk.full.geography.brma@1`, `uk.full.geography.rfdr@1` and `uk.full.private_rent@1` leave `microcosm.build.uk_runtime.graph_population`, with the helpers that composed them, the property quantile artifact type (`microcosm.uk.property-quantile-draw`) and the pool checkpoint's typed edge to it. `append_uk_population_nodes` no longer takes `property_localisation_mode`, `brma_localisation_mode`, `private_rent_localisation_mode`, `rfdr_localisation_mode` or `full_scale`, and `uk_pool_validation_inputs` no longer takes a property mode. On the atomic assignment the dense graph's geography is now the identity, the shared support imports, assignment and derivation, the local-authority resolve, the shared gate and the UK distribution gate. +- **Nodes and kernels.** `uk.full.property.value@1`, `uk.full.property.band@1`, `uk.full.geography.brma@1`, `uk.full.geography.rfdr@1` and `uk.full.private_rent@1` leave `microcosm.build.uk_runtime.graph_population`, with the helpers that composed them, the property quantile artifact type (`microcosm.uk.property-quantile-draw`) and the pool checkpoint's typed edge to it. `append_uk_population_nodes` no longer takes `property_localisation_mode`, `brma_localisation_mode`, `private_rent_localisation_mode`, `rfdr_localisation_mode` or `full_scale`, and `uk_pool_validation_inputs` no longer takes a property mode. On the atomic assignment the dense graph's geography is now the identity, the shared support imports, assignment and derivation, the local-authority resolve, the shared gate and the UK distribution gate; the next change of #931 retires those too, when the dense line takes its areas from the spine. - **Configuration.** `UKFullBuildConfig` loses `property_localisation`, `brma_localisation`, `private_rent_localisation` and `rfdr_localisation`, and with them its refusal of BRMA target rows without area BRMAs. The dense line no longer leaves its BRMA-grain rows out on either assignment: they bind on the spine's `brma` column, which a spine the dense line accepts reads off its areas. So `append_uk_target_nodes` loses its `brma_rows` switch and the target selection's payload its `brma_rows_left_out` field, which nothing could set once the configuration field went. - **Flags.** `microcosm-build-uk` (`tools/build_uk_full.py`) no longer takes `--property-localisation`, `--price-paid-quantiles`, `--voa-band-mix`, `--scotgov-band-mix`, `--brma-localisation`, `--brma-area-layer`, `--private-rent-localisation`, `--brma-rent-quantiles`, `--rfdr-localisation` or `--rfdr-area-layer`, nor the six resource pins `--price-paid-quantiles-sha256`, `--voa-band-mix-sha256`, `--scotgov-band-mix-sha256`, `--brma-area-layer-sha256`, `--brma-rent-quantiles-sha256` and `--rfdr-area-layer-sha256`. A script that still passes one is refused as an unrecognized argument. The release-candidate refusals of the four `none` modes go with them: the spine-roster check holds every dense build, release candidates included, to a spine with the stages that set those columns. The build request's bindings lose their four localisation blocks, and the size-checkpoint identity loses the four blocks that bound those resources' pins. - **Kept.** Every function the nodes called stays, with its tests: `owner_uniforms`, `localise_values`, `localise_bands` and `spine_household_keys` in `property_localisation`, `area_brmas` in `brma_area_layer`, `area_flags` in `rfdr_area_layer` and `localise_rents` in `private_rent_localisation`. The spine's geography and private-rent transforms (`spine_geography`, `spine_private_rent`), which no stage declares yet, call `area_brmas`, `area_flags` and `localise_rents`, and the `regional_home_values` stage calls `owner_uniforms`. The committed resources the nodes read and the terminal `uk_local_brma_enum_domain` gate stay too, though no dataset build reads the VOA or Scottish band mix now. The unread mode tuples `BRMA_LOCALISATION_MODES`, `RFDR_LOCALISATION_MODES` and `PRIVATE_RENT_LOCALISATION_MODES` go; `PROPERTY_LOCALISATION_MODES` stays for now, since spine stages hash `property_localisation`. `private_rent_localisation`'s docstring now names the spine's private-rent stage as the caller that supplies the row keys and tie-break uniforms. `tools/uk_private_rent_pool.py` and `tools/uk_private_rent_receipt.py` now say they read only dense runs built before this change. diff --git a/changelog.d/931-dense-on-the-clone-first-spine.changed.md b/changelog.d/931-dense-on-the-clone-first-spine.changed.md new file mode 100644 index 0000000000..5508fe87f6 --- /dev/null +++ b/changelog.d/931-dense-on-the-clone-first-spine.changed.md @@ -0,0 +1,18 @@ +Build the dense line on the bound spine's clones, areas and sampling (#931). The dense line no longer samples, expands, keys or places anything of its own: its targets, measures and problem run on the spine's version as it stands, its pool is that version whole, and its clone count K, its sampling rung and its geography binding are the spine's. It still refuses every spine whose roster is not geography-first (`spine_posture`), so no dense build runs until the spine declares its area stages. + +- **Graph.** `uk.full.sample`, `uk.full.normalize`, `uk.full.expand` (with `uk.full.expand.owned`), `uk.full.identity`, `uk.full.locations`, `uk.full.geography_mapping`, the shared support imports, assignment, derivation and gate (`uk.full.geography.*`), `uk.full.geography.local_authority` and the UK distribution gate `uk.full.geography_gate` leave the dense graph, with their eight kernels and the population receipt, location draw and geography gate artifact types. `microcosm.build.uk_runtime.graph_population` keeps `context_frame`, `population_columns`, `population_slices` and the pool checkpoint, `uk.full.pool@1`, now a keep-all FILTER on `uk.full.spine_checkpoint` (`--input-h5`) or the spine's endpoint (`--spine-request`) with no typed edge, declared bitwise and hashing its own module and `national_frame`. `append_uk_population_nodes`, `uk_pool_validation_inputs`, `GEOGRAPHY_ASSIGNMENTS` and `UK_GEOGRAPHY_IDENTITY_INPUTS` go. The UK registry no longer registers the shared atomic-geography kernels; NZ's transport registers its own. The dense geography gate's ladder and installed-engine local-authority checks (#953) and the pool's refusal of a failed shared geography validation leave with their nodes; the spine's own geography gates take them over when they are registered later in #931, and until then the roster refusal holds every dense build. +- **Targets.** `append_uk_target_nodes` takes `population` with no default, and the full build passes the spine's version. The measures lose their edge to the dense geography gate and their check of it, and take the spine's transferred-gate admission the sample node held, where the graph declares the spine's gates: they refuse to read the version unless that stored report admits it, before they read anything else. +- **Configuration.** `UKFullBuildConfig` loses `sample_fraction`, `sample_seed`, `geography_assignment`, `source_vintage`, `constituency_vintage` and `source_lineage_modulus`. `n_clones` (default 1, the declared spine's) is the bound spine's K and `effective_sample_fraction` is `source_sample_fraction`, the spine's rung. `uk_full_graph` no longer takes `spine_weight_kind` or `atomic_geography_definition`, and refuses a composed spine whose `geographic_expand` node declares another K than the configuration's, naming both. `build_uk_country_graph` no longer takes `atomic_geography_definition`. +- **K, sampling and geography from the spine.** `prepare_full_build` reads them from the authenticated sidecar (`geographic_clone_count`, one where a sidecar predates the record; `sampling`; `release_posture`; the `geography` block) or from the spine request's `--n-clones`, `--sample-fraction` and `--sample-seed`, and sets the run's `n_clones`, `sample_fraction` and, where the spine was sampled, `sample_seed` to them before the run's parameters, its candidate identity digest, its size-checkpoint identity, its staging rung and both Logbook rungs record them. A spine whose sampling fraction is not a scale-ladder rung is refused there, naming the rungs. The attempt's build id, minted before preparation reads the spine, takes the spine's rung read ahead off the bound spine's sidecar or the spine request's `--sample-fraction` (f100 where that read fails, and preparation then reports the real error); preparation refuses an attempt whose build id carries another rung than the spine it binds, naming both rung tokens, and a refusal before the spine is bound records the build id's rung on its Logbook row. The request bindings gain a `spine` block (clone count, sampling and release posture), and their `geography` block is the spine's binding with its clone count beside it. The rowwise manifest reads its `sampling`, `identity.sampling`, rung and release verdict's full-rung leg from that spine block and its `geography.ladder_gate` from the export descriptor's geography check on the calibrated frame; its weight-kind chain names `geographic_expand` for the clones. +- **Bound-spine checks.** `_assert_spine_sidecar_binds_frame` also holds the sidecar's `geographic_clone_count` to the frame's clone count, naming both, and the bound spine's checkpoint kernel hashes `geographic_clones` for it. `spine_provenance_from_sidecar` adds the spine's clone count, `geography`, `sampling` and `release_posture`. The national build record, the national calibration diagnostics' build block and the dense manifest's `identity.spine.spine_provenance` carry that provenance, so they gain the four fields; a national build on a spine without clone indices records K = 1 and computes the same dataset. +- **Flags.** `microcosm-build-uk` no longer takes `--geography-assignment`, `--atomic-support-ew`, `--atomic-support-scotland`, `--atomic-support-ni`, `--atomic-support-sha256-ew`, `--atomic-support-sha256-scotland`, `--atomic-support-sha256-ni` or `--source-lineage-modulus`; a script that still passes one is refused as an unrecognized argument, on either role. The dense role refuses a `--sample-fraction` other than 1.0, any `--sample-seed`, `--candidate-clone-counts` with `--input-h5`, and `--n-clones` with `--candidate-clone-counts`, by name, before any input is read. `--n-clones` is an assertion of the spine's K: the dense role no longer fills it from the doctrine, and preparation refuses one that differs from the spine's, naming both, and on a spine request pointing at the request's own `--n-clones`. `--candidate-clone-counts` plans the spine request once per count, each as the request's own `--n-clones`. The engine blocks are held to one or the spine's K by the configuration; on validation only an asserted `--n-clones` is compared. `--expected-constituency-vintage` is still taken and recorded in the run's parameters, but its one reader, the legacy location draw, is gone, so no dense node checks the ladder against it; its help says so. +- **Release-candidate refusals.** A sampled spine is refused as a release candidate at preparation, where its rung is read, on either route. The release-candidate refusal of `--n-clones` other than the doctrine's leaves validation, as do the support pins' requirement and the legacy assignment's refusal; `--sample-fraction` other than 1.0 is refused on every dense build (above). The spine's clone count is held to the doctrine's, and its geography binding to the conditioned register, by later commits of #931. The ladder, Ledger and input pins, the target rule, the epochs, the measure exclusions, the local target levels and the signing key are refused as before. +- **The dense sampler.** `national_sampling` keeps the rung tokens, the default seed and the spine root's sampler; `sample_uk_spine_frame` and `uk_spine_source_family_units`, which drew whole source families for the dense line, go, with their re-exports. +- **Texts.** The row-wise lane's clone refusals (`rowwise_dataset`) point at the dense graph on a geography-first spine and promise no later group; `tools/build_uk_rowwise_dataset.py` says it serves spines without clone indices. `geographic_identity`, `household_lineage`, `geographic_clones` and `diagnostics` no longer describe the dense identity node as a key's writer or a column's reader: `household_lineage`'s three lineage refusals no longer open "Geography identity", its `UK_OPTIONAL_LINEAGE_COLUMNS`, which nothing reads now, goes, and `geographic_clones`' docstring names the national gates among the kernels that hash it. `tools/measure_uk_atomic_assignment.py`'s graph route, which ran the dense line's geography nodes, refuses with a pointer to the spine's own draw, its command before it reads any input, and the helpers that composed its requests go. The Orrery roster loses its geography group and the retired population nodes. +- **What moves.** Keys and records only; no dataset value. No spine stage's code changes: `household_lineage`, `geographic_clones` and `geographic_identity` change in docstrings, comments and refusal texts (and `household_lineage` loses a constant nothing reads), and `national_sampling` loses only the dense sampler, which no spine stage calls; the dense line still builds on no spine. On the synthetic H2 fixture the legacy oracle's final frame, stage records and stage evidence are identical before and after. + - Of the kernels the UK registries register, 84 remain (96 before: the eight dense population kernels and the four shared atomic-geography ones go), and 64 change hash: 39 of the spine registry's 58 (the root, `uk.create@1`; the 35 stages that hash `household_lineage`, `geographic_identity` among them; and `cgt_support_split`, `was_wealth` and `was_lisa`, which hash `geographic_clones` alone), and the 25 `uk.full` kernels that hash an edited module. `uk.spine-gates@1`, `uk.claim@1`, `uk.identity@1` and 17 stages keep theirs. A spine build's root also hashes `national_sampling` through its sampler. So every spine node re-keys, and every dense and national node after it. + - The pool was the last platform-bitwise UK kernel, so no dense node key carries the platform now (on the declared spine 16 of the dense graph's 23 nodes did); a shared graph store can serve them across platforms on their own bitwise declarations. + - Of the gate phase codes only the full battery's terminal code moves. + - A dense run's input pins no longer carry the three atomic-area supports, its recorded parameters drop `source_lineage_modulus`, and its size-checkpoint identity drops `source_lineage_modulus` and the `geography` block, so its input-pins digest, candidate identity digest and size-checkpoint identity change, and a size checkpoint cut before this change does not resume. The bound spine's provenance gains four fields (above). + - The coverage manifest's `--check` passes; no declaration, generated file, `gates.json` or schema changes. +- **Tests.** `test_uk_full_population_graph.py` and `test_uk_atomic_identity_graph.py` are removed with the nodes they ran, and `test_uk_national_sampling.py` now tests the root's sampler. Their tests of live code move: the lineage reader's branch paths, residential arms and refusals to the new `test_uk_household_lineage.py`, which runs `uk_household_clone_paths` itself, and the row-wise lane's clone and source-lineage refusals to `test_uk_rowwise_dataset.py`. The full-graph stand-in (`test_support/microcosm_build/uk_full_population_graph.py`) is a clone-first, geography-first bound spine: the toy households at K clones in the copy-id table's geographic field, keyed as `geographic_identity` keys them and placed on the toy supports with the shared operators, with a BRMA and a fuel-relief flag; a bound checkpoint of it takes a stand-in sidecar whose geography block is under the leaf's keys. New tests cover each retired flag, the dense role's spine-bound refusals on both routes, K, sampling and seed taken from the spine on both routes (a seven-clone spine without `--n-clones` records seven, in the parameters, the identity digest and a written manifest), a disagreeing `--n-clones`, a sidecar whose K is not its frame's, the engine blocks held to the spine's K, `--candidate-clone-counts` planning a spine request once per count, a release candidate on a sampled spine on both routes, a spine off the scale ladder, the build id's rung read ahead and held to the spine's, the measures' admission, the pool on the spine's version and its keep-all mask, the manifest's spine-bound sampling and ladder gate, the dense allow-list pin (every column a geography-first spine adds to the export surface is in `UK_ALLOWED_EXTRA_EXPORT_COLUMNS`), the local area support gate's row, effective-sample-size and source floors in the K-sensitive constants table, and the measurement tool's refusal. The `geographic_identity` stage's keys are held to `household_draw_key` itself. diff --git a/changelog.d/931-retire-legacy-national-sampler.removed.md b/changelog.d/931-retire-legacy-national-sampler.removed.md index 3642de9de5..32b98f2083 100644 --- a/changelog.d/931-retire-legacy-national-sampler.removed.md +++ b/changelog.d/931-retire-legacy-national-sampler.removed.md @@ -1 +1 @@ -Retire the UK national compact sampler, `sample_uk_national_frame`, and its source-family decoder, `uk_source_family_units`, from `microcosm.build.uk_runtime.national_sampling` (#931). The sampler drew rungs from the certified national compact. The decoder recovered each derivative row's source from retired id arithmetic: a clone multiplier of ten to the digit count of the largest canonical id, and SPI and capital-gains offsets of the largest raw and pre-capital-gains household ids plus one. No build or tool has called either since 8adc836b9 (#762) moved the row-wise candidate onto the spine sampler. Their private helpers go with them: the post-sample SPI replacement quota check, the required-column list and the clone-index column name. The module also leaves the copy-id tests' inline-rule allowlist, which now admits only `rowwise_geography` (the dense line's multiplier) and `copy_id_table` (its field spans). The spine sampler (`sample_uk_spine_frame`, `uk_spine_source_family_units`), the rung tokens and the default sampling seed are unchanged. The tests of the retired functions are removed, and the numeric-region refusal they covered is now tested on the spine sampler. No value moves. The full build's population nodes that hash the module (`uk.full.sample`, `uk.full.normalize`, `uk.full.expand`, `uk.full.locations`, `uk.full.geography_mapping`, `uk.full.pool` and `uk.full.geography_gate`) re-key, and with them every node downstream of `uk.full.sample`. No spine node re-keys, and no generated file moves. +Retire the UK national compact sampler, `sample_uk_national_frame`, and its source-family decoder, `uk_source_family_units`, from `microcosm.build.uk_runtime.national_sampling` (#931). The sampler drew rungs from the certified national compact. The decoder recovered each derivative row's source from retired id arithmetic: a clone multiplier of ten to the digit count of the largest canonical id, and SPI and capital-gains offsets of the largest raw and pre-capital-gains household ids plus one. No build or tool has called either since 8adc836b9 (#762) moved the row-wise candidate onto the spine sampler. Their private helpers go with them: the post-sample SPI replacement quota check, the required-column list and the clone-index column name. The module also leaves the copy-id tests' inline-rule allowlist, which now admits only `rowwise_geography` (the dense line's multiplier) and `copy_id_table` (its field spans). The spine sampler (`sample_uk_spine_frame`, `uk_spine_source_family_units`), the rung tokens and the default sampling seed are unchanged here; that sampler, which drew the dense line's source families, retires later in #931, when the dense line takes its sampling from the spine it binds. The tests of the retired functions are removed, and the numeric-region refusal they covered is now tested on the spine sampler. No value moves. The full build's population nodes that hash the module (`uk.full.sample`, `uk.full.normalize`, `uk.full.expand`, `uk.full.locations`, `uk.full.geography_mapping`, `uk.full.pool` and `uk.full.geography_gate`) re-key, and with them every node downstream of `uk.full.sample`. No spine node re-keys, and no generated file moves. diff --git a/changelog.d/931-uk-atomic-geography-wiring.changed.md b/changelog.d/931-uk-atomic-geography-wiring.changed.md index 42edfd16e6..dbd45e16d1 100644 --- a/changelog.d/931-uk-atomic-geography-wiring.changed.md +++ b/changelog.d/931-uk-atomic-geography-wiring.changed.md @@ -1 +1 @@ -The UK full build assigns geography on the shared atomic-geography operators: households are keyed by their explicit spine lineage (`uk.full.identity`), one atomic area is drawn per household by census household count within FRS region under a keyed stream, larger geographies derive from the support mappings, and the shared gate must pass before the pool checkpoint. `--geography-assignment legacy` keeps the sequential ladder draw for measurement builds; release candidates refuse it and pin the three supports (#931). +The UK full build assigns geography on the shared atomic-geography operators: households are keyed by their explicit spine lineage (`uk.full.identity`), one atomic area is drawn per household by census household count within FRS region under a keyed stream, larger geographies derive from the support mappings, and the shared gate must pass before the pool checkpoint. `--geography-assignment legacy` keeps the sequential ladder draw for measurement builds; release candidates refuse it and pin the three supports (#931). Later in #931 the dense line retires these nodes and flags and takes its geographic clones, areas and sampling from the spine it binds, whose own stages assign them. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 90411f338a..e16b066d6c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -402,10 +402,6 @@ validate_uk_national_frame, write_uk_national_frame, ) -from microcosm.build.uk_runtime.national_sampling import ( - sample_uk_spine_frame, - uk_spine_source_family_units, -) from microcosm.build.uk_runtime.oa_ladder_sources import ( LADDER_OA_COLUMNS, assemble_uk_oa_ladder, @@ -623,7 +619,6 @@ "rowwise_calibration_mass_reason", "runtime_provenance", "rotated_uk_local_holdout", - "sample_uk_spine_frame", "uk_cgt_annual_exempt_amount", "UK_CGT_REQUIRED_COLUMNS", "UK_CGT_TARGET_COVERAGE_REQUIREMENTS", @@ -937,7 +932,6 @@ "uk_calibration_diagnostics_payload", "uk_fit_by_family", "uk_support_limited_misses", - "uk_spine_source_family_units", "uk_frame_content_identity", "uk_dsa_policy", "uk_dwp_disability_category_rates", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index f8f8495620..940d9f4f4b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -61,6 +61,7 @@ UK_SPINE_GATE_SCOPE, uk_scoped_gate_manifest, ) +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count # Moved to ledger_surface, which the UK target kernels hash, so an edit # elsewhere in this module no longer re-keys them (microcosm#1492). Kept @@ -426,6 +427,14 @@ def _assert_spine_sidecar_binds_frame( sidecar: Mapping[str, object], frame: Frame, ) -> None: + """Refuse a sidecar that describes another frame than the one bound. + + The row counts, the household weight kind and total, and the geographic + clone count K (microcosm#931): a sidecar written before the spine recorded + its K describes a spine of one clone, and the frame's K is counted from + its clone index (``geographic_clone_count``). The refusals print counts, + the weight kind and total and K, never an id or a value of a row. + """ expected_counts = sidecar.get("entity_row_counts") actual_counts = {entity: int(len(frame.table(entity))) for entity in frame.entities} if expected_counts != actual_counts: @@ -449,6 +458,15 @@ def _assert_spine_sidecar_binds_frame( "input H5 build sidecar household_weight_total mismatch: " f"sidecar {expected_total!r}, frame {actual_total!r}" ) + expected_clones = sidecar.get("geographic_clone_count", 1) + actual_clones = geographic_clone_count( + frame.table("household"), stage="The bound spine" + ) + if type(expected_clones) is not int or expected_clones != actual_clones: + raise ValueError( + "input H5 build sidecar geographic_clone_count mismatch: " + f"sidecar {expected_clones!r}, frame {actual_clones!r}" + ) def _spine_gate_report_path(sidecar_path: Path) -> Path: @@ -539,6 +557,16 @@ def spine_provenance_from_sidecar( # Empty for a spine built before the marker; the release assemblers # rederive it from the stages either way (microcosm#1095). "spi_public_use_tape": dict(sidecar.get("spi_public_use_tape") or {}), + # What the dense line takes from the spine it binds (microcosm#931): + # its clone count K (one where a sidecar predates the record), its + # geography binding (empty before the spine assigned areas), its + # sampling receipt (none at f100) and its release posture. + "geographic_clone_count": sidecar.get("geographic_clone_count", 1), + "geography": dict(sidecar.get("geography") or {}), + "sampling": ( + None if sidecar.get("sampling") is None else dict(sidecar["sampling"]) + ), + "release_posture": sidecar.get("release_posture"), } diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py index fccd45770b..37bbee09b9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py @@ -8,7 +8,6 @@ from __future__ import annotations import hashlib -from collections.abc import Mapping from dataclasses import replace from datetime import date from typing import TYPE_CHECKING @@ -172,13 +171,12 @@ def build_uk_country_graph( gate_review_date: date | None = None, release_candidate: bool = False, skip_holdout: bool = True, - atomic_geography_definition: Mapping | None = None, ) -> UKFullGraph: """Compile the canonical full graph; default target scope is all geographies. - ``atomic_geography_definition`` is the UK assignment declaration over the - admitted support bytes (``uk_atomic_assignment_definition``), required - while the config's ``geography_assignment`` is ``atomic`` (the default). + The dense line runs on the declared spine's version as it stands: the + spine's clones, areas and sampling are the build's, so the config's clone + count must be the spine's (one for the declared spine; microcosm#931). ``gate_review_date`` is the calibrated gates' clock, by default the solve clock ``review_date`` (microcosm#1301). """ @@ -218,7 +216,6 @@ def build_uk_country_graph( config, spine=spine, review_date=review_date.isoformat(), - atomic_geography_definition=atomic_geography_definition, ) graph = append_uk_full_gate_nodes( full.graph, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py index a5afc180b5..1dd368a015 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py @@ -514,8 +514,9 @@ def uk_weight_summary( #: Lineage columns that identify a support-split family: every copy the split #: created shares its root's source household, support channel, support clone -#: index and clone flag. These are the geography identity kernel's inputs -#: (microcosm#932) less the copy flag and index, so on the national spine only +#: index and clone flag. These are the first six lineage columns a household's +#: clone path is read from (``household_lineage.UK_HOUSEHOLD_LINEAGE_COLUMNS``) +#: less the support split's copy flag and index, so on the national spine only #: the split creates rows that agree on all four; on a K-expanded frame the #: geographic clone index (:func:`uk_support_family_geographic_clone_column`) #: joins the key, so each geographic clone folds only its own copies @@ -533,9 +534,8 @@ def uk_weight_summary( def uk_support_family_geographic_clone_column() -> str: - """The household geographic clone-index column: the spine's - ``geographic_expand`` stage writes it (microcosm#931), and ``uk.full.expand`` - on a spine built before that stage.""" + """The household geographic clone-index column, which the spine's + ``geographic_expand`` stage writes (microcosm#931).""" from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 30b3fc5ad3..a02f6d680d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -17,7 +17,10 @@ through :mod:`microcosm.build.uk_runtime.rowwise_cli`. The dense line builds only on a geography-first spine: :func:`prepare_full_build` refuses any other before it pins an input or composes a graph -(:mod:`microcosm.build.uk_runtime.spine_posture`, microcosm#931). +(:mod:`microcosm.build.uk_runtime.spine_posture`, microcosm#931), and takes the +spine's geographic clone count, sampling and areas as its own: its targets, +measures and pool run on the spine's version as it stands, and the run's +recorded clone count and rung are the spine's. ``tools/build_uk_rowwise_candidate.py`` and ``tools/build_uk_full.py`` are stubs over :func:`main`, so every UK line is built by this one command. @@ -38,7 +41,9 @@ from __future__ import annotations import argparse +import contextlib import hashlib +import io import json import sys import tempfile @@ -75,7 +80,6 @@ run_graph, ) from microcosm.graph.canonical import canonical_json -from microcosm.graph.codecs import load_raw_bytes from ..logbook_adoption import ( AttemptState, @@ -102,17 +106,7 @@ handover_record, refresh_sha256sums_entry, ) -from . import country_adapter, national_role -from .atomic_area_support import ( - IDENTITY_COLUMN, - uk_atomic_assignment_definition, -) -from .atomic_area_support import ( - SOURCES as ATOMIC_SUPPORT_SOURCES, -) -from .atomic_area_support import ( - SYSTEMS as ATOMIC_SUPPORT_SYSTEMS, -) +from . import national_role from .calibration_run import ( _write_json, load_bound_spine_checkpoint, @@ -158,7 +152,6 @@ solved_national_registry, uk_national_graph, ) -from .graph_population import GEOGRAPHY_ASSIGNMENTS from .graph_targets import TARGET_SELECTION_TYPE, registry_from_payload from .graph_terminal import ( FULL_DIAGNOSTICS_CSV_TYPE, @@ -265,11 +258,6 @@ #: local battery report takes the posture's ``*.local_gates.json`` name. FULL_GATE_REPORT_FILENAME = "uk.full.gates.calibrated.gate_report.json" -#: CLI labels for the three UK atomic-area support systems, in SYSTEMS order. -_SUPPORT_ARGUMENTS = dict( - zip(ATOMIC_SUPPORT_SYSTEMS, ("ew", "scotland", "ni"), strict=True) -) - def _run_graph_with_progress(compiled, *, retain_populations, execution=None, **kwargs): """Run a graph and emit one lightweight update per completed node. @@ -465,28 +453,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Full-UK OA geography ladder NPZ (required by the dense role).", ) parser.add_argument("--ladder-sha256", type=sha256_argument) - parser.add_argument( - "--geography-assignment", - choices=GEOGRAPHY_ASSIGNMENTS, - default="atomic", - help=( - "Dense role: atomic (default) is the identity-keyed single-stage " - "draw on the three UK atomic-area supports; legacy is the sequential " - "ladder draw, for measurement builds only." - ), - ) - for system, label in _SUPPORT_ARGUMENTS.items(): - parser.add_argument( - f"--atomic-support-{label}", - type=Path, - dest=f"atomic_support_{label}", - help=f"Atomic-area support NPZ for {system} (dense role, required with atomic).", - ) - parser.add_argument( - f"--atomic-support-sha256-{label}", - type=sha256_argument, - dest=f"atomic_support_sha256_{label}", - ) parser.add_argument( "--ledger-facts", type=Path, @@ -538,12 +504,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--n-clones", type=int, - help="Geographic pool copies K; defaults to the role's doctrine clone count.", + help=( + "Dense role: an assertion of the bound spine's geographic clone " + "count K, which the build takes from the spine; refused where it " + "differs from the spine's, and with --candidate-clone-counts." + ), ) parser.add_argument( "--candidate-clone-counts", type=candidate_clone_counts_argument, - help="Dry-run only comma-separated candidate clone counts.", + help=( + "Dry-run only, with --spine-request: comma-separated candidate " + "clone counts, each planned as the spine request's --n-clones." + ), ) parser.add_argument( "--dataset-households", @@ -554,10 +527,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--sample-fraction", type=float, default=1.0, - help="Optional pool sampling before cloning. Cannot resample an already sampled source spine.", + help=( + "Refused other than 1.0: the dense line takes its sampling from " + "the spine it binds; sample the spine at its own build." + ), ) parser.add_argument( - "--sample-seed", type=int, help="Dense role only; defaults to the spine seed." + "--sample-seed", + type=int, + help="Refused: the dense line takes its sampling seed from its spine.", ) parser.add_argument("--seed", type=int, help="Defaults to the role's seed.") parser.add_argument("--selection-seed", type=int) @@ -660,10 +638,13 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--engine-blocks", type=int, default=1) parser.add_argument("--source-year", type=int) parser.add_argument("--calibration-year", type=int) - parser.add_argument("--source-lineage-modulus", type=int) parser.add_argument( "--expected-constituency-vintage", - help="Dense role only: constituency vintage required from the ladder.", + help=( + "Dense role only: recorded in the run's parameters. The dense line " + "takes its areas from the spine (microcosm#931), so no dense node " + "checks the ladder against it." + ), ) parser.add_argument( "--rotated-holdout", @@ -861,59 +842,6 @@ def _input_record(path: Path, pin: dict, *, pinned: bool) -> dict: } -def _prepare_geography( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> tuple[dict | None, dict]: - """Pin the three supports and declare the identity-keyed assignment. - - The definition is the UK adapter's declaration over the admitted support - bytes at the build seed; its sha256 (and every keyed draw) moves iff a - support artifact or the seed moves. Each support enters ``pins`` as a - flat pin role named by its graph source, so the Logbook input-pins - digest, the candidate identity digest and the size-checkpoint identity - all bind it. Legacy builds bind nothing here. - """ - if args.geography_assignment != "atomic": - return None, {"assignment": "legacy"} - # A release candidate builds on the registered supports (sources.yaml, - # the provenance register): the operator's pin proves the bytes are the - # ones requested, the register proves they are the ones reviewed. - register = ( - country_adapter.uk_atomic_support_register() - if getattr(args, "release_candidate", False) - else None - ) - support_pins = {} - payloads = {} - for system, label in _SUPPORT_ARGUMENTS.items(): - path = getattr(args, f"atomic_support_{label}") - if path is None: - raise ValueError( - "Atomic geography assignment requires --atomic-support-ew, " - "--atomic-support-scotland and --atomic-support-ni." - ) - pin = _pin(path, getattr(args, f"atomic_support_sha256_{label}")) - if register is not None and pin != register[system]: - raise ValueError( - f"--release-candidate requires the registered atomic-area support " - f"for --atomic-support-{label}: sha256 {pin['sha256']} " - f"({pin['size_bytes']} bytes) is not the sources.yaml row " - f"{register[system]['sha256']} ({register[system]['size_bytes']} bytes)." - ) - pins[ATOMIC_SUPPORT_SOURCES[system]] = pin - support_pins[system] = pin - payloads[system] = load_raw_bytes(path) - sources[ATOMIC_SUPPORT_SOURCES[system]] = path - definition = uk_atomic_assignment_definition(payloads, seed=args.seed) - return definition, { - "assignment": "atomic", - "definition_sha256": hashlib.sha256(canonical_json(definition)).hexdigest(), - "support_pins": support_pins, - "identity": IDENTITY_COLUMN, - "stream": list(definition["stream"]), - } - - @dataclass(frozen=True) class PreparedUKFullBuild: full: UKFullGraph @@ -1035,6 +963,175 @@ def _require_geography_first_spine(args: argparse.Namespace) -> None: require_geography_first_spine(stages, operations, from_sidecar=False) +def _spine_request_arguments(args: argparse.Namespace) -> list[str]: + """The spine request's command arguments, as the spine build parses them. + + ``--spine-h5`` is an output control of the checkpoint command: where the + request names none, one under ``--out`` is added. Preparation uses it only + for path configuration; the full graph writes its own H5. + """ + raw_arguments = json.loads(args.spine_request.read_text()) + if not isinstance(raw_arguments, list) or not all( + isinstance(x, str) for x in raw_arguments + ): + raise ValueError("The spine request must be a JSON array of command arguments.") + if "--spine-h5" not in raw_arguments and not any( + x.startswith("--spine-h5=") for x in raw_arguments + ): + raw_arguments += ["--spine-h5", str(args.out / "spine.h5")] + return raw_arguments + + +def _spine_rung_token(args: argparse.Namespace) -> str: + """The rung token a dense attempt's build id carries: its spine's. + + The id is minted before preparation reads the spine, so the rung is read + ahead, unauthenticated and for the label only: off the bound spine's build + sidecar (``--input-h5``) or the spine request's ``--sample-fraction``. + Where this early read cannot place the spine, the label is f100, and + preparation then reports the real error; it also refuses an attempt whose + label is not its bound spine's rung (microcosm#931). + """ + fraction: object = 1.0 + try: + if args.input_h5 is not None: + sidecar = _read_spine_sidecar( + Path(args.input_sidecar or args.input_h5.with_suffix(".build.json")) + ) + sampling = sidecar.get("sampling") or {} + if isinstance(sampling, Mapping): + fraction = float(sampling.get("fraction", 1.0)) + elif args.spine_request is not None: + from .spine_build import parse_uk_spine_args + + # The spine's own parser, quietly: a request it refuses exits + # through argparse, and preparation reports that refusal. + with contextlib.redirect_stderr(io.StringIO()): + fraction = float( + parse_uk_spine_args(_spine_request_arguments(args)).sample_fraction + ) + except (OSError, TypeError, ValueError, SystemExit): + fraction = 1.0 + return UK_SAMPLE_RUNG_TOKENS.get(fraction, UK_SAMPLE_RUNG_TOKENS[1.0]) + + +def _spine_sampling(fraction: float, seed: object) -> dict[str, object]: + """A spine's sampling as the dense build binds it: its rung, and the seed + it drew with (none at f100, where the root draws nothing). + + A fraction that is not a scale-ladder rung is refused here, before the + run records it, naming the rungs only (microcosm#931). + """ + fraction = float(fraction) + if fraction not in UK_SAMPLE_RUNG_TOKENS: + raise ValueError( + "The spine's sampling fraction is not a scale-ladder rung (one of " + + ", ".join(sorted(UK_SAMPLE_RUNG_TOKENS.values())) + + "): the dense line's rung is its spine's (microcosm#931)." + ) + sampled = fraction != 1.0 + return { + "fraction": fraction, + "seed": int(seed) if sampled and seed is not None else None, + "sampled": sampled, + "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], + } + + +def _bound_spine_binding(sidecar: Mapping[str, object]) -> dict[str, object]: + """What a bound spine's authenticated sidecar binds the dense build to. + + Its geographic clone count K (one where a sidecar predates the record, and + held to the frame's by the checkpoint authentication), its sampling (its + root's receipt, none at f100), its release posture and its geography block + with K beside it (microcosm#931). + """ + clones = int(sidecar.get("geographic_clone_count", 1)) + sampling = sidecar.get("sampling") or {} + return { + "geographic_clone_count": clones, + "sampling": _spine_sampling( + sampling.get("fraction", 1.0), sampling.get("seed") + ), + "release_posture": sidecar.get("release_posture"), + "geography": { + **dict(sidecar.get("geography") or {}), + "geographic_clone_count": clones, + }, + } + + +def _requested_spine_binding(raw: argparse.Namespace) -> dict[str, object]: + """What a spine request binds the dense build to: the request's + ``--n-clones``, ``--sample-fraction`` and ``--sample-seed``, and the + release posture the spine build's sidecar writer gives the same request + (microcosm#931). The in-graph spine writes no sidecar, so its geography + binding carries K alone.""" + clones = int(raw.n_clones) + return { + "geographic_clone_count": clones, + "sampling": _spine_sampling(raw.sample_fraction, raw.sample_seed), + "release_posture": ( + "non_release_smoke" + if raw.smoke + else "release_candidate" + if raw.release_candidate + else "development" + ), + "geography": {"geographic_clone_count": clones}, + } + + +def _without_spine_option(arguments: list[str], flag: str) -> list[str]: + """A spine request's arguments without ``flag`` and its value.""" + kept: list[str] = [] + skip = False + for item in arguments: + if skip: + skip = False + elif item == flag: + skip = True + elif not item.startswith(f"{flag}="): + kept.append(item) + return kept + + +def _bind_spine_arguments( + args: argparse.Namespace, + binding: Mapping[str, object], + *, + requested: bool = False, +) -> None: + """Make the run's clone count and sampling the bound spine's (microcosm#931). + + The dense line takes K, the sampling rung and its seed from the spine it + binds, so the recorded run parameters, the identity digest, the size + checkpoint's identity and the Logbook rung are the spine's. An explicit + ``--n-clones`` is an assertion of the spine's K: one that differs is + refused, naming both K, and on a spine request (``requested``) pointing + at the request's own ``--n-clones``. + """ + clones = int(binding["geographic_clone_count"]) + explicit = getattr(args, "_explicit_arguments", frozenset()) + if "n_clones" in explicit and args.n_clones != clones: + if requested: + raise ValueError( + f"--n-clones {args.n_clones} differs from the spine request's " + f"--n-clones K = {clones}: the dense line takes K from its " + "spine; set the clone count in the spine request (microcosm#931)." + ) + raise ValueError( + f"--n-clones {args.n_clones} differs from the bound spine's " + f"geographic clone count K = {clones}: the dense line takes K " + "from its spine (microcosm#931)." + ) + sampling = binding["sampling"] + args.n_clones = clones + args.sample_fraction = float(sampling["fraction"]) + if sampling["seed"] is not None: + args.sample_seed = int(sampling["seed"]) + + def prepare_full_build( args: argparse.Namespace, *, telemetry=None, attempt: dict | None = None ) -> PreparedUKFullBuild: @@ -1087,9 +1184,10 @@ def prepare_full_build( ) spine = bound_spine_graph(frame) endpoint = "uk.full.spine_checkpoint" - weight_kind = frame.weights_for("household").kind.value time_period = str(frame.metadata["time_period"]) - source_fraction = float((sidecar.get("sampling") or {}).get("fraction", 1.0)) + # The checkpoint authentication held the sidecar's clone count to the + # frame's, so either is the spine's K. + spine_binding = _bound_spine_binding(sidecar) stages = tuple(sidecar["stages"]) _rules_engine() # the uk extra must be installed; the identity is provenance engine_identity = hashlib.sha256( @@ -1105,19 +1203,16 @@ def prepare_full_build( "spine_provenance", endpoint, "spine_provenance", SPINE_PROVENANCE_TYPE ) else: - raw_arguments = json.loads(args.spine_request.read_text()) - if not isinstance(raw_arguments, list) or not all( - isinstance(x, str) for x in raw_arguments - ): - raise ValueError( - "The spine request must be a JSON array of command arguments." - ) - # --spine-h5 is an output control of the checkpoint command. Preparation - # uses it only for path configuration; the full graph writes its own H5. - if "--spine-h5" not in raw_arguments and not any( - x.startswith("--spine-h5=") for x in raw_arguments - ): - raw_arguments += ["--spine-h5", str(args.out / "spine.h5")] + raw_arguments = _spine_request_arguments(args) + # A dry run over candidate clone counts plans each one as the spine + # request's own K (microcosm#931). + candidate = getattr(args, "_candidate_clone_count", None) + if candidate is not None: + raw_arguments = [ + *_without_spine_option(raw_arguments, "--n-clones"), + "--n-clones", + str(int(candidate)), + ] raw = parse_uk_spine_args(raw_arguments) if args.release_candidate: raw.release_candidate = True @@ -1130,9 +1225,8 @@ def prepare_full_build( spine, kernels = prepared.graph, prepared.kernels sources.update(prepared.sources) endpoint = uk_spine_endpoint(spine).population - weight_kind = "importance" time_period = prepared.frs_release.time_period - source_fraction = raw.sample_fraction + spine_binding = _requested_spine_binding(raw) stages = prepared.stage_names engine_identity = prepared.engine_identity inputs["dataset"] = { @@ -1142,9 +1236,21 @@ def prepare_full_build( "pin_verified": False, "spine_request": str(args.spine_request.resolve()), } - # The atomic-area supports are admitted after the spine so a spine - # refusal (sidecar, digest) is reported before a support one. - definition, geography_binding = _prepare_geography(args, pins, sources) + # The attempt's build id was minted with the spine's rung, read ahead of + # this; one that is not the bound spine's is refused, naming both rung + # tokens (microcosm#931). + labelled = getattr(args, "_build_id_rung", None) + rung = spine_binding["sampling"]["rung_token"] + if labelled is not None and labelled != rung: + raise ValueError( + f"The attempt's build id carries rung {labelled}, but the spine it " + f"binds is at rung {rung}: a dense build id takes its spine's rung " + "(microcosm#931)." + ) + # The spine's clone count, sampling and geography binding are the build's + # (microcosm#931): they enter the recorded run parameters, and so the + # identity digest, before anything records them. + _bind_spine_arguments(args, spine_binding, requested=args.input_h5 is None) stage( telemetry, "input_pinning", @@ -1162,16 +1268,10 @@ def prepare_full_build( target_families=("census_households/constituency",) if args.households_only else None, - n_clones=args.n_clones, - sample_fraction=args.sample_fraction, - source_sample_fraction=source_fraction, - sample_seed=args.sample_seed, + n_clones=int(spine_binding["geographic_clone_count"]), + source_sample_fraction=float(spine_binding["sampling"]["fraction"]), seed=args.seed, engine_blocks=args.engine_blocks, - constituency_vintage=args.expected_constituency_vintage, - source_lineage_modulus=args.source_lineage_modulus, - geography_assignment=args.geography_assignment, - source_vintage=release.vintage, calibration=UKGraphCalibrationConfig( epochs=args.epochs, learning_rate=args.learning_rate, @@ -1199,6 +1299,8 @@ def prepare_full_build( state.identity_digest = candidate_identity_digest( pins=pins, args=args, source_year=config.source_year ) + # The spine's rung (its sidecar's sampling, or the request's + # --sample-fraction): the dense line samples nothing more (microcosm#931). if args.release_candidate and config.effective_sample_fraction != 1.0: raise ValueError( "Sampled builds cannot request release-candidate certification." @@ -1244,11 +1346,9 @@ def prepare_full_build( config, spine=spine, spine_population=endpoint, - spine_weight_kind=weight_kind, optional_target_sources=tuple(optional), review_date=args.review_date.isoformat(), checkpoint_identity=checkpoint_identity, - atomic_geography_definition=definition, ) if args.resume_size_checkpoint: from .size_checkpoint import ( @@ -1302,7 +1402,13 @@ def prepare_full_build( }, "paired_ladder_sha256": pins["ladder"]["sha256"], }, - "geography": geography_binding, + # The spine the build binds and its geography binding (microcosm#931): + # the rowwise manifest reads its sampling and areas from here. + "spine": { + key: spine_binding[key] + for key in ("geographic_clone_count", "sampling", "release_posture") + }, + "geography": dict(spine_binding["geography"]), } graph = add_uk_export_preparation( graph, @@ -1572,10 +1678,6 @@ def _argument_sources(args: argparse.Namespace) -> dict[str, Path]: ("native_scorecard", args.native_scorecard), ("matched_size_scorecard", args.matched_size_scorecard), ("resume_size_checkpoint", args.resume_size_checkpoint), - *( - (f"atomic_support_{label}", getattr(args, f"atomic_support_{label}")) - for label in _SUPPORT_ARGUMENTS.values() - ), ) if path is not None } @@ -1947,6 +2049,8 @@ def _close_attempt( disposition=classified.disposition, predecessor=attempt["predecessor"], spool_dir=output / "logbook-spool", + # The spine's rung: preparation set the run's fraction to the spine's + # (microcosm#931). rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], ) print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) @@ -3639,24 +3743,33 @@ def _record_failure( predecessor=attempt["predecessor"], base_dir=args.out.resolve(), spool_dir=args.out.resolve() / "logbook-spool", - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + # The attempt's rung, its build id's: the spine's, read ahead of + # preparation and held there to the bound spine's, so a refusal + # before the spine is bound records it too (microcosm#931). + rung=getattr(args, "_build_id_rung", None) + or UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], pipeline=pipeline, disposition=disposition, ) def _dry_run(args: argparse.Namespace) -> int: - """Plan without solving or writing; loop over candidate clone counts.""" + """Plan without solving or writing. + + With ``--candidate-clone-counts`` (a spine request only), the spine + request is planned once per candidate, each as its own ``--n-clones``: + the dense line's K is its spine's (microcosm#931). + """ counts = args.candidate_clone_counts if counts is None: return execute_full_build(prepare_full_build(args), args) inventories = [] for count in counts: planned = argparse.Namespace(**vars(args)) - planned.n_clones = int(count) + planned._candidate_clone_count = int(count) prepared = prepare_full_build(planned) inventories.append( - {"n_clones": int(count), **prepared.full.operation_inventory()} + {"n_clones": int(planned.n_clones), **prepared.full.operation_inventory()} ) print(json.dumps(inventories, indent=2)) return 0 @@ -3683,10 +3796,11 @@ def _main(argv: list[str] | None = None) -> int: raise ValueError("--candidate-clone-counts is valid only with --dry-run.") started_at = time.perf_counter() started_ts = datetime.now(UTC) + # The id carries the spine's rung, read ahead of preparation, which + # refuses an attempt whose label is not its bound spine's (microcosm#931). + args._build_id_rung = _spine_rung_token(args) build_id = new_candidate_build_id( - seed=args.seed, - timestamp=started_ts, - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + seed=args.seed, timestamp=started_ts, rung=args._build_id_rung ) return _run_with_telemetry( args, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py index 89afe885f0..fc2a2c6df1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py @@ -22,9 +22,11 @@ module with the counting modules (``graph_kernels._COUNTING_MODULES``). The CGT support split, the support-clip stages and property components read ``K`` through the count, and the WAS stages, which count nothing by lineage, -hash this module as a helper. The full build's population and export -kernels, which read the names, hash it as well. So an edit here re-keys all -of them, without binding the dense line's ``rowwise_dataset`` or +hash this module as a helper. The bound spine's checkpoint, which holds its +sidecar's ``K`` to its frame's count, the national gates, whose +support-family fold reads the names, and the full build's export kernels, +which read the names, hash it as well. So an edit here re-keys all of them, +without binding the dense line's ``rowwise_dataset`` or ``atomic_area_support`` into the spine (microcosm#931). """ @@ -69,9 +71,8 @@ #: imports nothing first-party). GEOGRAPHIC_COPY_ID_FIELD = "geographic_support" #: Each clone's geography key, which ``geographic_identity`` writes on every -#: household of the spine and the dense line's ``uk.full.identity`` on every -#: household of its pool (``atomic_area_support.IDENTITY_COLUMN`` is this -#: name). +#: household of the spine, the dense line's pool among them +#: (``atomic_area_support.IDENTITY_COLUMN`` is this name). GEOGRAPHY_HOUSEHOLD_KEY_COLUMN = "geography_household_key" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py index cb7ecdf719..f30a6d2033 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py @@ -2,10 +2,10 @@ G1b of the clone-first plan runs right after the geographic expansion (:mod:`~microcosm.build.uk_runtime.geographic_expand`). It gives every -household the key the area assignment draws on, ``geography_household_key``, -as the dense line's ``uk.full.identity`` writes it: ``household_draw_key`` -over the FRS source, the household's FRS vintage, its true source and its -copy path. +household the key the area assignment draws on, ``geography_household_key``: +``household_draw_key`` over the FRS source, the household's FRS vintage, its +true source and its copy path, the key the dense line's own identity node gave +its pool before the dense line took its clones from the spine (microcosm#931). - **Path.** The shared reader (``source_lineage.copy_paths`` on ``household_lineage.UK_COPY_LINEAGE``) reads it from the lineage columns, @@ -15,15 +15,15 @@ ``source_year`` (:func:`frs_vintages_by_source_year`). A year it pins no vintage for is refused. - **Dtype.** The key is written in the graph's ``string`` dtype (pandas' - ``StringDtype("python")``), as ``uk.full.identity`` writes it, so it is not - a legacy object-string column. The step names the dtype itself, as - ``atomic_household_lineage`` does, rather than through the graph's token - table: a stage's transform computes through the Frame's modules alone, never - the executor's. + ``StringDtype("python")``), the dtype the graph stores a ``string`` cell + at, so it is not a legacy object-string column. The step names the dtype + itself, as ``atomic_household_lineage`` does, rather than through the + graph's token table: a stage's transform computes through the Frame's + modules alone, never the executor's. -So clone 0's key is the key the dense identity gives the same base household, -clone ``k``'s the key it gives that household's clone ``k``, and growing K -never changes an existing household's key. +So clone 0's key is the key the same rule gives the base household, clone +``k``'s the key it gives that household's clone ``k``, and growing K never +changes an existing household's key. Refusals give counts, columns, dtypes and declared constants only, never a key, an id or a year of a row. The evidence gives the distinct source @@ -85,9 +85,8 @@ ] GEOGRAPHIC_IDENTITY_STAGE_NAME = "geographic_identity" -#: The source every key names: the dense line's identity source (the default -#: ``identity_source`` of ``graph_population.append_uk_population_nodes``), so -#: clone 0 keys as the dense line keys a base household. +#: The source every key names: the FRS, as the dense line's own identity node +#: named it before the dense line took its keys from the spine (microcosm#931). GEOGRAPHIC_IDENTITY_SOURCE = "frs" #: The household columns the step reads. GEOGRAPHIC_IDENTITY_INPUTS = ( @@ -131,9 +130,9 @@ def frs_vintages_by_source_year( ``source_year`` from the frame's time period, the release's base year, as the SPI support channel did before that stage took the column over (and still does on a household table without the clone index). So both years - map to the vintage (one year when they are equal), as the dense identity - keys every household with it; any other year maps to none. ``release`` - defaults to the packaged pin. + map to the vintage (one year when they are equal), and every household's + key names it; any other year maps to none. ``release`` defaults to the + packaged pin. """ release = load_uk_frs_release() if release is None else release diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py index 2b376ab810..830c07e1c4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -2,13 +2,15 @@ The same graph handles all targets (the default), explicit target filters, dense exports and exact-count exports. A bound spine checkpoint resumes this -composition; an arbitrary historical uk-data H5 is not a build source. +composition; an arbitrary historical uk-data H5 is not a build source. The +full build composes its targets, measures and problem on the spine's own +version, and its pool is that version whole: the spine's geographic clones, +areas and sampling are the build's (microcosm#931). """ from __future__ import annotations import json -from collections.abc import Mapping from dataclasses import asdict, dataclass, field, replace from pathlib import Path @@ -49,65 +51,48 @@ from .graph_kernels import UKClaimKernel, UKIdentityKernel from .graph_population import ( UKPoolCheckpointKernel, - append_uk_population_nodes, population_columns, population_slices, register_uk_population_kernels, - uk_pool_validation_inputs, ) from .graph_targets import append_uk_target_nodes, register_uk_target_kernels -from .local_doctrine import UK_LOCAL_CLONE_COUNT -from .national_sampling import UK_SAMPLE_SEED_DEFAULT SPINE_PROVENANCE_TYPE = ArtifactType("microcosm.uk.bound-spine-provenance", 1) @dataclass(frozen=True) class UKFullBuildConfig: - """Three independent controls: geography target scope, pool K, export k.""" + """Two independent controls, geography target scope and export k, over the + bound spine. + + The spine binds the rest (microcosm#931): ``n_clones`` is its geographic + clone count K and ``source_sample_fraction`` its sampling rung, both read + from the spine the build binds, never chosen here. The full build samples, + expands and places nothing of its own. + """ calibration_year: int time_period: str = field(default_factory=lambda: load_uk_frs_release().time_period) source_year: int = field(default_factory=lambda: load_uk_frs_release().survey_year) geography_levels: tuple[str, ...] | None = None target_families: tuple[str, ...] | None = None - n_clones: int = UK_LOCAL_CLONE_COUNT - sample_fraction: float = 1.0 + n_clones: int = 1 source_sample_fraction: float = 1.0 - sample_seed: int = UK_SAMPLE_SEED_DEFAULT seed: int = 42 engine_blocks: int = 1 - constituency_vintage: str = "2024_pcon" - source_lineage_modulus: int | None = None calibration: UKGraphCalibrationConfig = UKGraphCalibrationConfig() - #: ``atomic`` keys every post-clone household and draws on the shared - #: atomic-geography operators; ``legacy`` keeps the sequential ladder draw - #: for measurement builds (a release candidate refuses it). - geography_assignment: str = "atomic" - #: FRS release vintage folded into every geography identity key. - source_vintage: str = field(default_factory=lambda: load_uk_frs_release().vintage) def __post_init__(self) -> None: - from .graph_population import GEOGRAPHY_ASSIGNMENTS - from .national_sampling import validate_sample_fraction + from microcosm.build.frame_sampling import validate_sample_fraction - if self.geography_assignment not in GEOGRAPHY_ASSIGNMENTS: - raise ValueError( - f"Geography assignment must be one of {GEOGRAPHY_ASSIGNMENTS}." - ) - if not isinstance(self.source_vintage, str) or not self.source_vintage: - raise ValueError("The FRS source vintage must be a non-empty string.") - validate_sample_fraction(self.sample_fraction, label="UK full pool") validate_sample_fraction(self.source_sample_fraction, label="UK source spine") - if self.sample_fraction != 1.0 and self.source_sample_fraction != 1.0: + if type(self.n_clones) is not int or self.n_clones < 1: raise ValueError( - "A sampled spine cannot be sampled a second time in the full build." + "The spine's geographic clone count K must be a positive integer." ) - if type(self.n_clones) is not int or self.n_clones < 1: - raise ValueError("Geographic pool K must be a positive integer.") if self.engine_blocks not in {1, self.n_clones}: raise ValueError( - "Engine blocks must be one or equal the geographic pool K." + "Engine blocks must be one or equal the spine's geographic clone count K." ) if self.geography_levels is not None: if not self.geography_levels or set(self.geography_levels) - { @@ -137,7 +122,8 @@ def __post_init__(self) -> None: @property def effective_sample_fraction(self) -> float: - return self.sample_fraction * self.source_sample_fraction + """The spine's sampling rung: the dense line samples nothing more.""" + return self.source_sample_fraction @dataclass(frozen=True) @@ -172,22 +158,37 @@ def operation_inventory(self) -> dict: } +def _declared_spine_clone_count(graph: Graph) -> int | None: + """The clone count K a composed spine's ``geographic_expand`` node declares. + + ``None`` where the graph composes no such node, as a bound checkpoint does: + its K is read from its frame and build sidecar where it is bound. + """ + + from .geographic_expand import GEOGRAPHIC_EXPAND_STAGE_NAME + + for node in graph.nodes: + if node.id == GEOGRAPHIC_EXPAND_STAGE_NAME: + return int(node.params["geographic_clones"]) + return None + + def uk_full_graph( config: UKFullBuildConfig, *, spine: Graph | None = None, spine_population: str | None = None, - spine_weight_kind: str = "importance", optional_target_sources: tuple[str, ...] = (), checkpoint_identity: dict | None = None, review_date: str | None = None, - atomic_geography_definition: Mapping | None = None, ) -> UKFullGraph: """Append the full build to the existing source-owned UK spine graph. - ``atomic_geography_definition`` is the validated UK assignment declaration - (``uk_atomic_assignment_definition``) and is required when the config's - ``geography_assignment`` is ``atomic``. + The targets, measures and problem run on the spine's own version, and the + pool checkpoint keeps every row of it: the dense line takes the spine's + geographic clones, areas and sampling as they stand (microcosm#931). A + composed spine whose ``geographic_expand`` node declares another clone + count than the config's ``n_clones`` is refused, naming both K. """ initial = uk_spine_graph(source_mode="split") if spine is None else spine @@ -196,24 +197,16 @@ def uk_full_graph( if spine_population is None else spine_population ) - graph = append_uk_population_nodes( + declared = _declared_spine_clone_count(initial) + if declared is not None and declared != config.n_clones: + raise ValueError( + f"The full build is configured for K = {config.n_clones} geographic " + f"clones, but the spine it composes makes K = {declared}; the " + "dense line takes K from its spine (microcosm#931)." + ) + graph = append_uk_target_nodes( initial, population=endpoint, - time_period=config.time_period, - weight_kind=spine_weight_kind, - sample_fraction=config.sample_fraction, - sample_seed=config.sample_seed, - n_clones=config.n_clones, - seed=config.seed, - source_year=config.source_year, - constituency_vintage=config.constituency_vintage, - source_lineage_modulus=config.source_lineage_modulus, - geography_assignment=config.geography_assignment, - atomic_geography_definition=atomic_geography_definition, - source_vintage=config.source_vintage, - ) - graph = append_uk_target_nodes( - graph, calibration_year=config.calibration_year, time_period=config.time_period, geography_levels=config.geography_levels, @@ -224,10 +217,11 @@ def uk_full_graph( optional_sources=optional_target_sources, review_date=review_date, ) - cells = population_columns(graph, "uk.full.expand") - # This structural checkpoint depends on every pool operation, including - # evidence-only gates and the contribution problem. It binds the complete - # incoming ledger before downstream nodes reconstruct any Frame slices. + cells = population_columns(graph, endpoint) + # This structural checkpoint depends on every operation on the spine's + # version, the evidence-only gates and the contribution problem among + # them. It binds the complete incoming ledger before downstream nodes + # reconstruct any Frame slices, and keeps every row. graph = replace( graph, nodes=( @@ -236,10 +230,9 @@ def uk_full_graph( "uk.full.pool", UKPoolCheckpointKernel.ref, structural=StructuralDelta.FILTER, - base="uk.full.expand", + base=endpoint, inputs=population_slices(cells), - artifact_inputs=uk_pool_validation_inputs(config.geography_assignment), - description="Checkpoint the complete geographic pool and selected-target problem.", + description="Checkpoint the bound spine and the selected-target problem.", ), ), ) @@ -287,13 +280,20 @@ def implementation_hash(self) -> str: from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from .. import gate_battery, gates, trace - from . import content_identity, gate_scopes, graph_frames, hmrc_uprating + from . import ( + content_identity, + gate_scopes, + geographic_clones, + graph_frames, + hmrc_uprating, + ) # The node also computes through the H5 it loads (frame_materialize, # the engine adapter's contracts), the checkpoint's content identity - # and pinned engine (content_identity, hmrc_uprating), the gate - # digests it derives through a battery run (gate_battery, gates, - # trace) over the spine gate scope and its scoped manifest + # and pinned engine (content_identity, hmrc_uprating), the frame's + # geographic clone count it holds the sidecar's to (geographic_clones), + # the gate digests it derives through a battery run (gate_battery, + # gates, trace) over the spine gate scope and its scoped manifest # (gate_scopes), the create-frame normalisation it shares with the # spine's root (graph_frames, not the spine's kernel module, so an # edit there does not re-key the national line) and the canonical @@ -303,6 +303,7 @@ def implementation_hash(self) -> str: national_frame, calibration_run, gate_scopes, + geographic_clones, release_certification, frame_materialize, uk_engine_adapter, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index 44d66a9aed..8e7e7197aa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -1,25 +1,24 @@ -"""Population operations in the single UK full-build graph. - -Source interpretation stays in UK adapters. Selection, row ancestry, typed -weights, content storage and replay are enforced by the shared graph runtime. -The legacy numerical functions remain the only implementations of the draws. +"""The UK full build's pool checkpoint and the population helpers its nodes share. + +The dense line builds only on a geography-first spine +(:mod:`~microcosm.build.uk_runtime.spine_posture`), as that spine stands: its +``geographic_expand`` and ``geographic_identity`` stages give every household +its geographic clones and keys, its geography stages assign each clone its +area, and its root samples at the spine's rung, so the full build samples, +expands, keys and places nothing of its own (microcosm#931). Its targets and +measures read the spine's version, and :class:`UKPoolCheckpointKernel` keeps +every row of it before the solve. Selection, row ancestry, typed weights, +content storage and replay are enforced by the shared graph runtime. """ from __future__ import annotations -import importlib.util -import json from collections.abc import Mapping -from dataclasses import asdict, replace -import numpy as np import pandas as pd -from microcosm.frame import Frame, Weights +from microcosm.frame import Frame from microcosm.graph import ( - ArtifactInput, - ArtifactOutput, - ArtifactType, Capabilities, Determinism, Graph, @@ -27,68 +26,15 @@ KernelContext, KernelRegistry, KernelResult, - Node, Numeric, - Owned, - SeedSource, Slice, - SourceRef, StructuralDelta, - WeightUpdate, compile_graph, source_hash, ) -from microcosm.graph.canonical import canonical_json -from microcosm.graph.population import dtype_for_token -from microcosm.graph.weight_update import weight_update_receipt -from .. import frame_sampling, gates -from . import ( - atomic_area_support, - atomic_household_identity, - geographic_clones, - geography_ladder, - local_authority_input, - national_frame, - national_sampling, - rowwise_dataset, - rowwise_geography, - spi_support, -) -from .geographic_clones import ladder_clone_index_column -from .geography_ladder import ( - UK_GEOGRAPHY_LADDER_COLUMNS, - derive_uk_ladder_locations, - draw_uk_ladder_locations, - load_uk_oa_ladder, -) -from .household_lineage import ( - UK_HOUSEHOLD_LINEAGE_COLUMNS, - UK_OPTIONAL_LINEAGE_COLUMNS, - uk_household_clone_paths, -) -from .local_authority_input import ( - UK_LOCAL_AUTHORITY_CODE_COLUMN, - UK_LOCAL_AUTHORITY_INPUT_COLUMN, - UK_LOCAL_AUTHORITY_VINTAGE, - resolve_local_authority_engine_keys, - verify_local_authority_engine_domain, -) +from . import national_frame from .national_frame import UK_NATIONAL_SCHEMA -from .rowwise_dataset import expand_uk_geographic_pool - -POPULATION_RECEIPT_TYPE = ArtifactType("microcosm.uk.population-receipt", 1) -LOCATION_DRAW_TYPE = ArtifactType("microcosm.uk.ladder-location-draw", 1) -GEOGRAPHY_GATE_TYPE = ArtifactType("microcosm.uk.geography-gate", 1) -GEOGRAPHY_ASSIGNMENTS = ("atomic", "legacy") -#: Lineage columns the post-clone household identity is keyed on. The support -#: channel, the CGT support split's copy flag and index (microcosm#1045) and -#: the clone flag come from the spine stages, as do the source id and the -#: geographic clone index on a clone-first spine, whose ``geographic_expand`` -#: stage writes them (microcosm#931); on a spine built before that stage, -#: ``uk.full.expand`` writes the index. Calibration K, weights and remapped ids -#: are not inputs. -UK_GEOGRAPHY_IDENTITY_INPUTS = UK_HOUSEHOLD_LINEAGE_COLUMNS def context_frame(context: KernelContext) -> Frame: @@ -136,804 +82,42 @@ def population_slices(cells: Mapping[tuple[str, str], str]) -> tuple[Slice, ...] ) -def _series(frame: Frame, entity: str, column: str, dtype: str) -> pd.Series: - table = frame.table(entity) - ids = pd.Index( - table[frame.schema.entity_id_column(entity)], - name=frame.schema.entity_id_column(entity), - ) - return pd.Series(table[column].array, index=ids, name=column).astype( - dtype_for_token(dtype) - ) - - -def _mass_records(before: Frame, after: Frame) -> list[dict]: - if after.mass_log[: len(before.mass_log)] != before.mass_log: - raise ValueError( - "A graph population operation replaced its incoming mass ledger." - ) - return [asdict(record) for record in after.mass_log[len(before.mass_log) :]] - - -def _declared_mass(before: Frame, after: Frame) -> dict: - old, new = before.stratum_mass(), after.stratum_mass() - return { - "policy": "declared", - "before": float(old.sum()), - "after": float(new.sum()), - "stratum_before": {k: float(v) for k, v in old.items()}, - "stratum_after": {k: float(v) for k, v in new.items()}, - } - - -class _PopulationKernel(KernelBase): - def implementation_hash(self) -> str: - from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter - - from . import graph_evidence - - # The dense population nodes also compute through what these call: - # the sample's household draw and mass normalisation (frame_sampling), - # its strata columns (spi_support's replacement strata), the clones' - # minted ids and copied tables (rowwise_geography), the clone index - # columns' names (geographic_clones), the national frame they rebuild - # and the engine adapter's contracts it stamps on them, the local - # authority names the geography resolves, the gate's results, their - # artifacts' canonical encoding and the normalisation's weight-update - # receipt (microcosm#931); test_uk_graph derives the set from what the - # kernels name. - return source_hash( - type(self), - geography_ladder, - national_sampling, - rowwise_dataset, - geographic_clones, - graph_evidence, - frame_sampling, - gates, - local_authority_input, - national_frame, - rowwise_geography, - spi_support, - uk_engine_adapter, - canonical_json, - weight_update_receipt, - ) - - -class UKFamilySampleKernel(_PopulationKernel): - ref = "uk.full.sample@1" - capabilities = Capabilities( - Determinism.DETERMINISTIC, - seed_source=SeedSource.PARAM, - structural=StructuralDelta.FILTER, - ) - - def run(self, context: KernelContext) -> KernelResult: - from .graph_evidence import require_uk_spine_gate_admission - - require_uk_spine_gate_admission(context) - before = context_frame(context) - fraction = float(context.params["fraction"]) - seed = int(context.params["seed"]) - if fraction == 1.0: - after = before - receipt = { - "fraction": fraction, - "seed": seed, - "sampled": False, - "pre_household_count": before.n("household"), - "post_household_count": before.n("household"), - "rung_token": "f100", - } - else: - after, receipt = national_sampling.sample_uk_spine_frame( - before, fraction=fraction, seed=seed - ) - receipt = {"sampled": True, **receipt} - ids = before.table("person")["person_id"] - keep = pd.Series( - ids.isin(after.table("person")["person_id"]).to_numpy(), - index=pd.Index(ids, name="person_id"), - dtype=bool, - ) - # The sampling function computes the historical normalization once. - # Installing its weights is a separate declared same-kind operation. - payload = { - "receipt": receipt, - "household_ids": after.table("household")["household_id"].tolist(), - "weights": after.weights_for("household").values.tolist(), - "mass_log_append": _mass_records(before, after), - } - return KernelResult(keep=keep, artifacts={"sampling": canonical_json(payload)}) +class UKPoolCheckpointKernel(KernelBase): + """Checkpoint the bound spine before the solve, keeping every row. - -class UKSampleNormalizationKernel(_PopulationKernel): - ref = "uk.full.normalize@1" - capabilities = Capabilities( - Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT - ) - - def run(self, context: KernelContext) -> KernelResult: - frame = context_frame(context) - payload = json.loads(context.artifacts["sampling"].payload) - ids = frame.table("household")["household_id"].tolist() - if ids != payload["household_ids"]: - raise ValueError("Sample normalization household axis changed.") - weights = Weights( - np.asarray(payload["weights"], dtype=np.float64), - kind=context.weights["household"].kind, - ) - after = Frame( - {e: frame.table(e) for e in frame.entities}, - frame.schema, - {"household": weights}, - frame.strata, - metadata=frame.metadata, - ) - return KernelResult( - weights=weights, - receipt={ - "weight_update": weight_update_receipt(ids), - "mass": _declared_mass(frame, after), - "frame_mass_log_append": payload["mass_log_append"], - }, - ) - - -class UKGeographicExpansionKernel(_PopulationKernel): - ref = "uk.full.expand@1" - capabilities = Capabilities( - Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND - ) - - def run(self, context: KernelContext) -> KernelResult: - before = context_frame(context) - household = before.table("household").copy() - household["household_weight"] = before.weights_for("household").values - pool = expand_uk_geographic_pool( - person=before.table("person"), - benunit=before.table("benunit"), - household=household, - n_clones=int(context.params["n_clones"]), - source_year=int(context.params["source_year"]), - time_period=str(context.params["time_period"]), - household_weight_kind=before.weights_for("household").kind, - mass_log=before.mass_log, - source_lineage_modulus=context.params.get("source_lineage_modulus"), - ) - after = pool.frame - ancestry = {} - for entity in before.entities: - table = after.table(entity) - id_column = before.schema.entity_id_column(entity) - added = table.iloc[before.n(entity) :] - # One clone adds no row. A spine whose copy ids reach the copy-id - # table's top fields (microcosm#931) has a multiplier above int64, - # and multiplying even an empty int64 column by it raises. - source_ids = ( - np.empty(0, dtype=np.int64) - if added.empty - else added[id_column].to_numpy() - - added[ladder_clone_index_column(entity)].to_numpy() - * pool.id_multiplier - ) - ancestry[entity] = pd.Series( - source_ids, index=pd.Index(added[id_column], name=id_column) - ) - columns = { - (e, c): _series(after, e, c, dtype) - for e, c, dtype in context.params["expand_cells"] - } - return KernelResult( - expand=ancestry, - columns=columns, - weights=after.weights_for("household"), - artifacts={ - "expansion": canonical_json( - {"n_clones": pool.n_clones, "id_multiplier": pool.id_multiplier} - ) - }, - receipt={"frame_mass_log_append": _mass_records(before, after)}, - ) - - -class UKLocationDrawKernel(_PopulationKernel): - ref = "uk.full.locations@1" - capabilities = Capabilities(Determinism.DETERMINISTIC, seed_source=SeedSource.PARAM) - - def run(self, context: KernelContext) -> KernelResult: - ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) - household = context.tables["household"] - indices = draw_uk_ladder_locations( - household, - ladder, - seed=int(context.params["seed"]), - expected_constituency_vintage=str(context.params["constituency_vintage"]), - ) - return KernelResult( - artifacts={ - "locations": canonical_json( - { - "household_ids": household["household_id"].tolist(), - "indices": indices.tolist(), - "seed": int(context.params["seed"]), - "layer_vintages": ladder.layer_vintages, - "constituency_sampling_basis": ladder.metadata[ - "constituency_sampling_basis" - ], - "oa_sampling_basis": ladder.metadata["oa_sampling_basis"], - } - ) - } - ) - - -class UKGeographyMappingKernel(_PopulationKernel): - ref = "uk.full.geography_mapping@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - - def run(self, context: KernelContext) -> KernelResult: - payload = json.loads(context.artifacts["locations"].payload) - household = context.tables["household"] - if household["household_id"].tolist() != payload["household_ids"]: - raise ValueError("Location draw is bound to a different household axis.") - assigned = derive_uk_ladder_locations( - household, - load_uk_oa_ladder(context.sources["uk_ladder"]), - np.asarray(payload["indices"], dtype=np.int64), - ) - index = pd.Index(household["household_id"], name="household_id") - return KernelResult( - columns={ - (owned.entity, owned.column): pd.Series( - assigned[owned.column].array, index=index, name=owned.column - ).astype(dtype_for_token(owned.dtype)) - for owned in context.node.outputs - } - ) - - -class UKGeographyIdentityKernel(_PopulationKernel): - """Key every post-clone household by its source id and ordered clone path. - - The path is read from the spine's explicit lineage columns, never inferred - from ids, weights or financial values: ``spi_support_channel`` with the - support clone index when the household is an SPI support copy, - ``cgt_support_split`` with the copy index when it is a support copy - (microcosm#1045), ``cgt_incidence_clone`` (ordinal 1) from its flag, - ``geographic_support`` with the pool clone index when it is not the - original copy. Growing K never changes an existing household's key. - """ - - ref = "uk.full.identity@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - - def implementation_hash(self) -> str: - from microcosm.frame import concepts as frame_concepts - - from .. import source_lineage - from . import household_lineage - - # The clone paths are read through the lineage readers, on the clone - # index column geographic_clones names and the default source column - # frame_concepts names, and the draw keys are encoded canonically - # (microcosm#931). - return source_hash( - type(self), - atomic_household_identity, - atomic_area_support, - rowwise_dataset, - geographic_clones, - household_lineage, - source_lineage, - frame_concepts, - canonical_json, - ) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - source = str(context.params["source"]) - vintage = str(context.params["source_vintage"]) - source_ids, paths = uk_household_clone_paths(household) - keys = [ - atomic_household_identity.household_draw_key( - source=source, - source_vintage=vintage, - source_household_id=source_id, - clone_path=path, - ) - for source_id, path in zip(source_ids, paths, strict=True) - ] - if len(set(keys)) != len(keys): - raise ValueError("Geography identity keys are not unique.") - index = pd.Index(household["household_id"], name="household_id") - column = atomic_area_support.IDENTITY_COLUMN - return KernelResult( - columns={ - ("household", column): pd.Series( - keys, index=index, name=column, dtype=dtype_for_token("string") - ) - }, - receipt={ - "households": len(keys), - "source": source, - "source_vintage": vintage, - "inputs": list(UK_GEOGRAPHY_IDENTITY_INPUTS), - }, - ) - - -class UKLocalAuthorityResolveKernel(_PopulationKernel): - """Resolve the engine's ``local_authority`` input from the derived code. - - The shared derive step writes ``local_authority_code``; the engine input - is the member name resolved fail-closed on the April 2023 roster - (microcosm#953), exactly as the legacy ladder derive writes it. The - engine itself is never imported here; the UK distribution gate checks - the assigned names against the installed enum. - """ - - ref = "uk.full.geography.local_authority@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - - def implementation_hash(self) -> str: - return source_hash(type(self), local_authority_input) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - keys = resolve_local_authority_engine_keys( - household[UK_LOCAL_AUTHORITY_CODE_COLUMN] - ) - index = pd.Index(household["household_id"], name="household_id") - column = UK_LOCAL_AUTHORITY_INPUT_COLUMN - return KernelResult( - columns={ - ("household", column): pd.Series( - keys, index=index, name=column, dtype=dtype_for_token("string") - ) - }, - receipt={ - "households": int(len(keys)), - "local_authority_vintage": UK_LOCAL_AUTHORITY_VINTAGE, - }, - ) - - -class UKPoolCheckpointKernel(_PopulationKernel): - """Checkpoint the complete pool; refuse unless the shared atomic gate passed. - - The shared ``geography.gate@1`` result is a platform-scoped typed artifact - (amendment 17), and a bitwise UK kernel consuming it as a typed input - would be refused. This FILTER node consumes it and emits no typed - artifact of its own, so that refusal is avoided here. It is only the - typed-artifact edge that stops: the shared assignment kernels are - ``platform_bitwise`` too, and their columns reach the UK target and - calibration chain through population slices, which the executor already - reads at platform scope. Whether the shared kernels can honestly declare - ``bitwise`` (exact keyed draws, integer inverse CDF) is the open question - recorded on microcosm#932. + A keep-all FILTER on the spine's version: the dense pool is the bound + spine, its geographic clones and areas with it (microcosm#931). The node's + base is that version, so it follows every operation on it, the target + compilation, selection, measures and problem among them, and it binds the + complete incoming ledger before the calibration nodes reconstruct any + Frame slices. It reads and emits no typed artifact. """ ref = "uk.full.pool@1" capabilities = Capabilities( Determinism.DETERMINISTIC, - numeric=Numeric.PLATFORM_BITWISE, + numeric=Numeric.BITWISE, structural=StructuralDelta.FILTER, ) + def implementation_hash(self) -> str: + # The checkpoint reads the person axis national_frame's schema names, + # and nothing else (microcosm#931). + return source_hash(type(self), national_frame) + def run(self, context: KernelContext) -> KernelResult: - receipt = {"scope": "complete_pool_checkpoint"} - if "atomic_validation" in context.artifacts: - validation = json.loads(context.artifacts["atomic_validation"].payload) - if validation.get("outcome") != "pass": - raise ValueError("Shared atomic geography validation did not pass.") - receipt["atomic_validation"] = validation person = context.tables[UK_NATIONAL_SCHEMA.person_entity] id_column = UK_NATIONAL_SCHEMA.person_id_column ids = pd.Index(person[id_column].to_numpy(copy=True), name=id_column) return KernelResult( - keep=pd.Series(True, index=ids, dtype="bool"), receipt=receipt - ) - - -def uk_pool_validation_inputs(geography_assignment: str) -> tuple[ArtifactInput, ...]: - """The typed edge that orders the shared atomic gate before the checkpoint.""" - if geography_assignment != "atomic": - return () - from microcosm.build.graph_atomic_geography import ( - ATOMIC_GEOGRAPHY_VALIDATION_TYPE, - ) - - return ( - ArtifactInput( - "atomic_validation", - "uk.full.geography.gate", - "validation", - ATOMIC_GEOGRAPHY_VALIDATION_TYPE, - ), - ) - - -class UKGeographyGateKernel(_PopulationKernel): - ref = "uk.full.geography_gate@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - - def run(self, context: KernelContext) -> KernelResult: - frame = context_frame(context) - household = frame.table("household") - gate = geography_ladder.uk_geography_ladder_gate( - household, frame.weights_for("household").values - ) - # Fail closed against the installed engine (microcosm#953): every - # assigned local_authority member name must exist in this pin's - # LocalAuthority enum. Only assigned keys are checked so a toy ladder - # stays evaluable and an engine-free lane never imports the engine. - if gate.passed and importlib.util.find_spec("policyengine_uk") is not None: - verify_local_authority_engine_domain(household["local_authority"].unique()) - # Persist the failure. Downstream target materialization explicitly - # refuses this outcome; cached failure must never turn into a pass. - payload = asdict(gate) - return KernelResult( - artifacts={"gate": canonical_json(payload)}, - receipt={"outcome": "pass" if gate.passed else "fail", "evidence": payload}, - ) - - -def append_uk_population_nodes( - graph: Graph, - *, - population: str, - time_period: str, - weight_kind: str, - sample_fraction: float = 1.0, - sample_seed: int = national_sampling.UK_SAMPLE_SEED_DEFAULT, - n_clones: int = 1, - seed: int = 42, - source_year: int = 2024, - constituency_vintage: str = "2024_pcon", - source_lineage_modulus: int | None = None, - geography_assignment: str = "atomic", - atomic_geography_definition: Mapping | None = None, - source_vintage: str | None = None, - identity_source: str = "frs", -) -> Graph: - """Compose sampling → replication → identity → assignment → derivation → gate. - - ``geography_assignment="atomic"`` (the default) keys every post-clone - household by its lineage and runs the shared atomic-geography operators - on the three UK support artifacts declared by ``atomic_geography_definition`` - (``uk_atomic_assignment_definition``). ``"legacy"`` keeps the sequential - ladder draw for measurement builds only. - """ - - from .graph_kernels import UKClaimKernel - - cells = population_columns(graph, population) - if type(n_clones) is not int or n_clones < 1: - raise ValueError("Geographic replicate count K must be a positive integer.") - if geography_assignment not in GEOGRAPHY_ASSIGNMENTS: - raise ValueError( - f"Geography assignment must be one of {GEOGRAPHY_ASSIGNMENTS}." - ) - if geography_assignment == "atomic": - if atomic_geography_definition is None: - raise ValueError( - "Atomic geography assignment requires the UK assignment definition." - ) - if not isinstance(source_vintage, str) or not source_vintage: - raise ValueError("Atomic geography assignment requires the FRS vintage.") - elif atomic_geography_definition is not None: - raise ValueError("Legacy geography assignment takes no atomic definition.") - national_sampling.validate_sample_fraction(sample_fraction, label="UK full build") - national_sampling.validate_sample_seed(sample_seed, label="UK full build") - common = {"time_period": str(time_period)} - spine_gate_inputs = () - spine_gate_params = {} - if any(node.id == "spine.gates.transferred" for node in graph.nodes): - from .graph_evidence import SPINE_GATE_REPORT_TYPE - - gate = graph.node("spine.gates.transferred") - spine_gate_inputs = ( - ArtifactInput("spine_gate", gate.id, "gate_report", SPINE_GATE_REPORT_TYPE), - ) - spine_gate_params = { - "spine_gate_phase": "transferred", - "spine_gate_release_candidate": bool(gate.params["release_candidate"]), - "spine_gate_synthetic_smoke": bool(gate.params["synthetic_smoke"]), - } - nodes = list(graph.nodes) - nodes.append( - Node( - id="uk.full.sample", - kernel=UKFamilySampleKernel.ref, - inputs=population_slices(cells), - base=population, - structural=StructuralDelta.FILTER, - mass="free", - params={ - **common, - "fraction": float(sample_fraction), - "seed": sample_seed, - **spine_gate_params, - }, - artifact_inputs=spine_gate_inputs, - artifact_outputs=(ArtifactOutput("sampling", POPULATION_RECEIPT_TYPE),), - description="Select intact source families; calculate their historical mass normalization.", - ) - ) - nodes.append( - Node( - id="uk.full.normalize", - kernel=UKSampleNormalizationKernel.ref, - inputs=population_slices(cells), - base="uk.full.sample", - structural=StructuralDelta.REWEIGHT, - mass="declared", - weights=WeightUpdate( - "household", weight_kind, "Normalize sampled source-family mass." - ), - artifact_inputs=( - ArtifactInput( - "sampling", "uk.full.sample", "sampling", POPULATION_RECEIPT_TYPE - ), - ), - params=common, - description="Install normalized weights without changing their kind.", + keep=pd.Series(True, index=ids, dtype="bool"), + receipt={"scope": "complete_pool_checkpoint"}, ) - ) - expansion_cells = { - ("household", "source_household_id"): "int64", - ("household", "source_year"): "int64", - ("household", "source_household_key"): "string", - **{ - (entity, ladder_clone_index_column(entity)): "int64" - for entity in UK_NATIONAL_SCHEMA.entities - }, - } - # Existing source lineage is carried unchanged, except the explicitly - # requested historical modulus conversion, which owns its declarations. - if source_lineage_modulus is not None: - expansion_cells.update( - {("household", rowwise_dataset.POOL_SOURCE_LINEAGE_COLUMN): "int64"} - ) - expansion_cells = { - key: cells.get(key, dtype) for key, dtype in expansion_cells.items() - } - nodes.append( - Node( - id="uk.full.expand", - kernel=UKGeographicExpansionKernel.ref, - inputs=population_slices(cells), - structural=StructuralDelta.EXPAND, - base="uk.full.normalize", - mass="conserve", - params={ - **common, - "source_year": source_year, - "n_clones": n_clones, - "source_lineage_modulus": source_lineage_modulus, - "expand_weight_entity": "household", - "expand_weight_kind": weight_kind, - "expand_cells": tuple( - (e, c, dtype) for (e, c), dtype in sorted(expansion_cells.items()) - ), - }, - artifact_outputs=(ArtifactOutput("expansion", POPULATION_RECEIPT_TYPE),), - description="Prepare source lineage and expand linked entities into K geographic copies.", - ) - ) - nodes.append( - Node( - id="uk.full.expand.owned", - kernel=UKClaimKernel.ref, - population="uk.full.expand", - outputs=tuple( - Owned(e, c, dtype, rewrite=(e, c) in cells) - for (e, c), dtype in sorted(expansion_cells.items()) - ), - params={ - "materialized_expand_outputs": tuple( - f"{e}.{c}" for e, c in expansion_cells if (e, c) not in cells - ) - }, - description="Declare the lineage and replicate columns materialized by expansion.", - ) - ) - cells.update(expansion_cells) - if geography_assignment == "legacy": - geography_nodes, geography_sources = _legacy_geography_nodes( - cells, common=common, seed=seed, constituency_vintage=constituency_vintage - ) - else: - geography_nodes, geography_sources = _atomic_geography_nodes( - cells, - common=common, - seed=seed, - definition=atomic_geography_definition, - identity_source=identity_source, - source_vintage=source_vintage, - ) - nodes.extend(geography_nodes) - declared = {s.name for s in graph.sources} - sources = ( - *graph.sources, - *(ref for ref in geography_sources if ref.name not in declared), - ) - return replace(graph, nodes=tuple(nodes), sources=tuple(sources)) - - -def _legacy_geography_nodes( - cells: dict, *, common: Mapping, seed: int, constituency_vintage: str -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The sequential ladder draw, kept verbatim for measurement builds.""" - - nodes = ( - Node( - id="uk.full.locations", - kernel=UKLocationDrawKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", ("region",)),), - sources=("uk_ladder",), - params={"seed": seed, "constituency_vintage": constituency_vintage}, - artifact_outputs=(ArtifactOutput("locations", LOCATION_DRAW_TYPE),), - description="Draw constituency then atomic area with the current sequential RNG.", - ), - Node( - id="uk.full.geography_mapping", - kernel=UKGeographyMappingKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", ("region",)),), - sources=("uk_ladder",), - artifact_inputs=( - ArtifactInput( - "locations", "uk.full.locations", "locations", LOCATION_DRAW_TYPE - ), - ), - outputs=tuple( - Owned("household", col, "string", rewrite=("household", col) in cells) - for col in UK_GEOGRAPHY_LADDER_COLUMNS - ), - description="Derive every geography from the drawn atomic-area index.", - ), - ) - cells.update({("household", col): "string" for col in UK_GEOGRAPHY_LADDER_COLUMNS}) - gate = Node( - id="uk.full.geography_gate", - kernel=UKGeographyGateKernel.ref, - population="uk.full.expand", - inputs=population_slices(cells), - params=dict(common), - artifact_outputs=(ArtifactOutput("gate", GEOGRAPHY_GATE_TYPE),), - description="Validate geography integrity before selected-target contributions.", - ) - ladder = SourceRef( - "uk_ladder", "raw-bytes-v1", "Full UK atomic-area ladder with pinned vintages." - ) - return (*nodes, gate), (ladder,) - - -def _atomic_geography_nodes( - cells: dict, - *, - common: Mapping, - seed: int, - definition: Mapping, - identity_source: str, - source_vintage: str, -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """Identity-keyed assignment on the shared operators (release path order).""" - - from microcosm.build.atomic_geography import validate_assignment_spec - from microcosm.build.graph_atomic_geography import atomic_geography_nodes - - spec = validate_assignment_spec(definition) - if spec["identity"] != [atomic_area_support.IDENTITY_COLUMN]: - raise ValueError("UK atomic assignment must key on the geography identity.") - if spec["stream"][3] != seed: - raise ValueError("UK atomic assignment seed differs from the build seed.") - if {system["source"] for system in spec["systems"]} != set( - atomic_area_support.SOURCES.values() - ): - raise ValueError("UK atomic assignment must declare the three UK supports.") - lineage_inputs = tuple( - sorted( - column - for column in (*UK_HOUSEHOLD_LINEAGE_COLUMNS, *UK_OPTIONAL_LINEAGE_COLUMNS) - if ("household", column) in cells - ) - ) - # A clone-first spine already keys its households (geographic_identity), - # so there the node rewrites the key on the pool (microcosm#931). - identity_cell = ("household", atomic_area_support.IDENTITY_COLUMN) - identity = Node( - id="uk.full.identity", - kernel=UKGeographyIdentityKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", lineage_inputs),), - outputs=( - Owned( - "household", - atomic_area_support.IDENTITY_COLUMN, - "string", - rewrite=identity_cell in cells, - ), - ), - params={ - **common, - "source": identity_source, - "source_vintage": source_vintage, - }, - description="Key every post-clone household by its source id and ordered clone path.", - ) - cells[("household", atomic_area_support.IDENTITY_COLUMN)] = "string" - shared = atomic_geography_nodes( - spec, - tuple(Owned(e, c, dtype) for (e, c), dtype in sorted(cells.items())), - base="uk.full.expand", - prefix="uk.full.geography", - emit_validation_artifact=True, - ) - for node in shared: - cells.update({(o.entity, o.column): o.dtype for o in node.outputs}) - # The engine's local_authority enum input (microcosm#953) is UK-side: the - # shared derive writes the GSS code, this node resolves the member name - # the export gate and the engine require, between derive and the gates. - local_authority = Node( - id="uk.full.geography.local_authority", - kernel=UKLocalAuthorityResolveKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", (UK_LOCAL_AUTHORITY_CODE_COLUMN,)),), - outputs=(Owned("household", UK_LOCAL_AUTHORITY_INPUT_COLUMN, "string"),), - params=dict(common), - description="Resolve the engine local_authority input from the derived GSS code.", - ) - cells[("household", UK_LOCAL_AUTHORITY_INPUT_COLUMN)] = "string" - # The shared gate's typed validation artifact is platform-scoped, so it is - # consumed by the pool checkpoint (uk.full.pool, no typed outputs), not by - # this distribution gate, whose own artifact feeds the bitwise UK chain. - gate = Node( - id="uk.full.geography_gate", - kernel=UKGeographyGateKernel.ref, - population="uk.full.expand", - inputs=population_slices(cells), - params=dict(common), - artifact_outputs=(ArtifactOutput("gate", GEOGRAPHY_GATE_TYPE),), - description="Validate the UK geography distribution on the identity-keyed assignment.", - ) - sources = ( - SourceRef( - "uk_ladder", - "raw-bytes-v1", - "Full UK atomic-area ladder with pinned vintages.", - ), - *( - SourceRef( - atomic_area_support.SOURCES[system], - "raw-bytes-v1", - f"UK atomic-area support artifact for {system}.", - ) - for system in atomic_area_support.SYSTEMS - ), - ) - return (identity, *shared[:-1], local_authority, shared[-1], gate), sources def register_uk_population_kernels(registry: KernelRegistry) -> None: - from microcosm.build.graph_atomic_geography import ( - register_atomic_geography_kernels, - ) + """Register the pool checkpoint. The shared atomic-geography kernels are + not the UK's to register: a geography-first spine assigns its areas in its + own stages, and NZ's transport registers the shared kernels it runs.""" - for kernel in ( - UKFamilySampleKernel(), - UKSampleNormalizationKernel(), - UKGeographicExpansionKernel(), - UKGeographyIdentityKernel(), - UKLocalAuthorityResolveKernel(), - UKLocationDrawKernel(), - UKGeographyMappingKernel(), - UKGeographyGateKernel(), - UKPoolCheckpointKernel(), - ): - registry.register(kernel) - register_atomic_geography_kernels(registry) + registry.register(UKPoolCheckpointKernel()) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index 6e15957c07..dec32408f4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -65,7 +65,6 @@ from .full_targets import CHRONICLE_SOURCE_CODEC, load_chronicle_source_bytes from .geography_ladder import load_uk_oa_ladder, uk_area_region_codes from .graph_population import ( - GEOGRAPHY_GATE_TYPE, context_frame, population_columns, population_slices, @@ -537,11 +536,14 @@ def implementation_hash(self) -> str: ).hexdigest() def run(self, context: KernelContext) -> KernelResult: - gate = json.loads(context.artifacts["geography_gate"].payload) - if not gate["passed"]: - raise ValueError( - "Geography integrity gate failed: " + "; ".join(gate["failures"]) - ) + from .graph_evidence import require_uk_spine_gate_admission + + # The spine's transferred gates admit the measures before they read + # the spine's version, where the composed spine declares them + # (microcosm#931). The dense line's own geography gate retired with + # its area draw; the spine's geography gates, registered later in + # #931, check the areas. + require_uk_spine_gate_admission(context) frame = context_frame(context) full = json.loads(context.artifacts["surface"].payload) selected = registry_from_payload( @@ -870,7 +872,7 @@ def run(self, context: KernelContext) -> KernelResult: def append_uk_target_nodes( graph: Graph, *, - population: str = "uk.full.expand", + population: str, calibration_year: int, time_period: str, geography_levels: tuple[str, ...] | None = None, @@ -881,7 +883,13 @@ def append_uk_target_nodes( optional_sources: tuple[str, ...] = (), review_date: str | None = None, ) -> Graph: - """Default all geographies. Scope is independent of K, k and solver outcomes.""" + """Default all geographies. Scope is independent of K, k and solver outcomes. + + ``population`` is the spine's version the targets read, the full build's + pool (microcosm#931). Where the graph declares the spine's transferred + gates, the measures take their admission: they refuse to read the version + unless that stored report admits it. + """ if geography_levels is not None and not geography_levels: raise ValueError("An explicit geography selector must contain levels.") @@ -891,6 +899,20 @@ def append_uk_target_nodes( slices = population_slices(cells) compile_sources = ("uk_ladder", "uk_ledger_facts", *optional_sources) common = {"time_period": time_period} + spine_gate_inputs: tuple[ArtifactInput, ...] = () + spine_gate_params: dict[str, object] = {} + if any(node.id == "spine.gates.transferred" for node in graph.nodes): + from .graph_evidence import SPINE_GATE_REPORT_TYPE + + gate = graph.node("spine.gates.transferred") + spine_gate_inputs = ( + ArtifactInput("spine_gate", gate.id, "gate_report", SPINE_GATE_REPORT_TYPE), + ) + spine_gate_params = { + "spine_gate_phase": "transferred", + "spine_gate_release_candidate": bool(gate.params["release_candidate"]), + "spine_gate_synthetic_smoke": bool(gate.params["synthetic_smoke"]), + } review_date = date.today().isoformat() if review_date is None else review_date date.fromisoformat(review_date) # The solve fingerprint, as the target kernels hash it (microcosm#1301). @@ -932,17 +954,8 @@ def append_uk_target_nodes( UKFullMeasureKernel.ref, population=population, inputs=slices, - params={**common, "engine_blocks": engine_blocks}, - artifact_inputs=( - surface, - selection, - ArtifactInput( - "geography_gate", - "uk.full.geography_gate", - "gate", - GEOGRAPHY_GATE_TYPE, - ), - ), + params={**common, "engine_blocks": engine_blocks, **spine_gate_params}, + artifact_inputs=(surface, selection, *spine_gate_inputs), artifact_outputs=(ArtifactOutput("measures", MEASURE_TYPE),), description="Evaluate selected measures and compile temporary national contributions.", ), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 1548687246..ff6b2fd2f5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -1924,12 +1924,6 @@ def rowwise_candidate_manifest_from_graph( holdout = json.loads( _graph_payload(final_manifest, store, "uk.full.holdout", "holdout") ) - geography_gate = json.loads( - _graph_payload(final_manifest, store, "uk.full.geography_gate", "gate") - ) - sampling = json.loads( - _graph_payload(final_manifest, store, "uk.full.sample", "sampling") - )["receipt"] size_receipt = _optional_graph_json( final_manifest, store, "uk.full.size_refit", "size" ) @@ -1939,17 +1933,33 @@ def rowwise_candidate_manifest_from_graph( ) or {} ) - # The request's geography binding (assignment mode, definition digest, - # support pins, identity column, stream) travels in the export descriptor. + # The request's bindings travel in the export descriptor: the spine the + # build binds (its clone count, its sampling and its release posture) and + # the spine's geography binding (assignment, identity column, definition + # digest, support pins, stream, register and clone count), since the + # dense line takes its pool, sampling and areas from the spine + # (microcosm#931). The ladder gate is the descriptor's own geography + # integrity check, on the calibrated release frame. export_descriptor = ( _optional_graph_json( final_manifest, store, "uk.full.export.prepare", "export_descriptor" ) or {} ) - geography_binding = dict( - (export_descriptor.get("bindings") or {}).get("geography") or {} - ) + request_bindings = dict(export_descriptor.get("bindings") or {}) + geography_binding = dict(request_bindings.get("geography") or {}) + spine_binding = dict(request_bindings.get("spine") or {}) + if not isinstance(spine_binding.get("sampling"), Mapping): + raise ValueError( + "The export descriptor's request binds no spine sampling: the dense " + "line's rung is its spine's (microcosm#931)." + ) + sampling = dict(spine_binding["sampling"]) + spine_fraction = float(sampling["fraction"]) + ladder_gate = { + **dict(export_descriptor.get("geography_integrity") or {}), + "phase": "post_calibration", + } enforcement = dict(gate_document["enforcement"]) gate_rows = _gate_rows(gate_document) local_scope = { @@ -1973,7 +1983,7 @@ def rowwise_candidate_manifest_from_graph( ) size_drift = size_doctrine_drift(args) releasable, release_posture = release_verdict( - sample_fraction=args.sample_fraction, + sample_fraction=spine_fraction, engine_blocks=args.engine_blocks, release_blocking_gates_passed=bool( enforcement["release_blocking_gates_passed"] @@ -2043,8 +2053,7 @@ def rowwise_candidate_manifest_from_graph( "uk.full.gates.calibrated", "uk.full.holdout", "uk.full.package", - "uk.full.sample", - "uk.full.geography_gate", + "uk.full.export.prepare", "uk.full.size_refit", "uk.full.spine_checkpoint", } @@ -2098,8 +2107,8 @@ def rowwise_candidate_manifest_from_graph( "sampling": dict(sampling), "rung_surface": { **dict(bindings.get("rung_surface", {})), - "rung": UK_SAMPLE_RUNG_TOKENS[float(args.sample_fraction)], - "fraction": float(args.sample_fraction), + "rung": UK_SAMPLE_RUNG_TOKENS[spine_fraction], + "fraction": spine_fraction, "unreachable_check": "completed", }, "outputs": { @@ -2125,20 +2134,20 @@ def rowwise_candidate_manifest_from_graph( ].nunique() ), "missing_geography_rows": 0, - "ladder_gate": {**geography_gate, "phase": "post_calibration"}, + "ladder_gate": dict(ladder_gate), "assignment": geography_binding, }, - "gate": {**geography_gate, "phase": "post_calibration"}, + "gate": dict(ladder_gate), "weights": { "household_weight_kind": weight_kind, "household_weight_kind_chain": [ {"stage": "staging", "kind": "importance"}, *( [] - if float(args.sample_fraction) == 1.0 + if spine_fraction == 1.0 else [{"stage": "sample", "kind": "importance"}] ), - {"stage": "ladder_clone", "kind": "importance"}, + {"stage": "geographic_expand", "kind": "importance"}, {"stage": "rowwise_calibration", "kind": weight_kind}, ], "mass_log_records_before_calibration": len(frame.mass_log) - 1, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py index 1b8ca6576d..8959982d19 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py @@ -1,14 +1,13 @@ """The ordered clone path of every post-clone UK household, from its lineage columns. -The geography identity kernel keys each pool household on its source id and -its explicit clone path (microcosm#931); the post-geography property node -(microcosm#1240) needs the same path without the geographic clone branch, so -that every pool clone of one spine household shares one key. Both read the -path through this module, which validates the lineage columns once and never -infers a branch from ids, weights or values. The UK's copy lineage is declared -here (:data:`UK_COPY_LINEAGE`) for the shared readers in -``microcosm.build.source_lineage``: the receipts' true sources and the -FRS-trained fits' collapse of geographic clones read it too, and +The spine's ``geographic_identity`` stage keys each household on its source id +and its explicit clone path, and the dense line keys no household of its own +(microcosm#931). The copy take-up stages read the same path through +:func:`uk_household_clone_paths`, which validates the lineage columns once and +never infers a branch from ids, weights or values. The UK's copy lineage is +declared here (:data:`UK_COPY_LINEAGE`) for the shared readers in +``microcosm.build.source_lineage``: the geography identity, the receipts' true +sources and the FRS-trained fits' collapse of geographic clones read it, and :func:`geographic_fold_receipt` records how a fit's fold met its survey records. """ @@ -47,13 +46,14 @@ "uk_household_clone_paths", ] -#: Lineage columns the post-clone household identity is keyed on. The support -#: channel, the CGT support split's copy flag and index (microcosm#1045), the -#: high-value home support copy flag and index (microcosm#1240) and the clone -#: flags come from the spine stages. So do the source id and the geographic -#: clone index, which the spine's ``geographic_expand`` stage writes -#: (microcosm#931); on a spine built before that stage, ``uk.full.expand`` -#: writes the index. Calibration K, weights and remapped ids are not inputs. +#: Lineage columns a post-clone household's clone path is read from. The +#: support channel, the CGT support split's copy flag and index +#: (microcosm#1045), the high-value home support copy flag and index +#: (microcosm#1240) and the clone flags come from the spine stages. So do the +#: source id and the geographic clone index, which the spine's +#: ``geographic_expand`` stage writes (microcosm#931); a spine built before +#: that stage carries no index. Calibration K, weights and remapped ids are +#: not inputs. HIGH_VALUE_SUPPORT_COPY_COLUMN = "household_is_high_value_support_copy" HIGH_VALUE_SUPPORT_INDEX_COLUMN = "high_value_support_copy_index" UK_HOUSEHOLD_LINEAGE_COLUMNS: tuple[str, ...] = ( @@ -67,12 +67,6 @@ "cgt_residential_clone_index", ladder_clone_index_column("household"), ) -#: The two optional columns the high-value support stage adds (microcosm#1240); -#: a spine built before that stage carries neither and keys as before. -UK_OPTIONAL_LINEAGE_COLUMNS: tuple[str, ...] = ( - HIGH_VALUE_SUPPORT_COPY_COLUMN, - HIGH_VALUE_SUPPORT_INDEX_COLUMN, -) GEOGRAPHIC_SUPPORT_BRANCH = "geographic_support" #: The evidence key under which a stage whose fit folds geographic clones #: records the fold (:func:`geographic_fold_receipt`), on a frame of more @@ -154,17 +148,20 @@ def uk_household_clone_paths( required = [c for c in UK_HOUSEHOLD_LINEAGE_COLUMNS if c != clone_column] missing = [c for c in required if c not in household] if missing: - raise ValueError(f"Geography identity lacks lineage column(s) {missing}.") + raise ValueError( + f"The household table lacks lineage column(s) {missing} that " + "its clone paths are read from." + ) present = [ column for column in (UK_COPY_LINEAGE.source, *UK_COPY_LINEAGE.copy_columns) if column in household ] if household.loc[:, present].isna().any().any(): - raise ValueError("Geography identity refuses null lineage values.") + raise ValueError("Household clone paths refuse null lineage values.") for column in _INT64_LINEAGE_COLUMNS: if column in household and household[column].dtype != np.dtype("int64"): - raise ValueError(f"Geography identity requires an int64 {column}.") + raise ValueError(f"Household clone paths require an int64 {column}.") sources, paths = copy_paths(household, UK_COPY_LINEAGE, require=_REQUIRED_BRANCHES) return [int(source) for source in sources], paths diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py index 842344c9aa..889e001e11 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py @@ -1,27 +1,16 @@ """Scale-ladder sampling for the UK spine (#627). -The UK full build samples the Microcosm spine at its ``uk.full.sample`` node, -before geographic expansion. Every derivative row on the spine names its raw -FRS household in ``source_household_id``, so the UK policy over the shared -:mod:`microcosm.build.frame_sampling` core is: - -- **The sampling unit is the source family**: the raw FRS household and - every row whose ``source_household_id`` names it. A drawn family is kept - whole, so a derived row never survives without its raw row; the families - are derived again on the sample, and a family left incomplete is refused. -- **The draw is stratified by the raw rows' region.** Derivative flags and - derivative rows' own regions never split a family across strata, and a - family whose raw rows span more than one region is refused. -- **Sampled mass is renormalized to the full-source household total** (the - US stacked semantics #627 names). - -A spine without lineage columns is sampled household by household, -stratified by region. The module also holds the rung tokens and the default -sampling seed that the UK builds share, and the spine build's root sampler: -a spine built below f100 samples its FRS root household by household +A spine built below f100 samples its FRS root household by household (:func:`sample_uk_spine_root`) through :class:`UKSampledSpineRootTransform`, which the root node hashes with this module, and below f100 with the shared -core too. +:mod:`microcosm.build.frame_sampling` core too. Sampled mass is renormalized +to the root's household total (the US stacked semantics #627 names). The +module also holds the rung tokens and the default sampling seed the UK builds +share. + +The dense line samples nothing of its own: it takes the bound spine's +sampling with its clones and areas (microcosm#931), so its sampler, which drew +whole source families from a spine, is retired. """ from __future__ import annotations @@ -29,20 +18,10 @@ import sys from collections.abc import Mapping -import numpy as np -import pandas as pd - from microcosm.build import frame_sampling from microcosm.build.frame_sampling import ( normalize_sampled_household_mass, sample_frame_households, - validate_sample_fraction, - validate_sample_seed, -) -from microcosm.build.uk_runtime.national_frame import validate_uk_national_frame -from microcosm.build.uk_runtime.spi_support import ( - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - SPI_REPLACEMENT_STRATA_COLUMNS, ) from microcosm.frame import Frame @@ -50,9 +29,7 @@ "UK_SAMPLE_RUNG_TOKENS", "UK_SAMPLE_SEED_DEFAULT", "UKSampledSpineRootTransform", - "sample_uk_spine_frame", "sample_uk_spine_root", - "uk_spine_source_family_units", ] #: The #624 scale-ladder rungs and their identity tokens, shared with the US @@ -67,225 +44,6 @@ #: from the build ``--seed`` so dev-scale seed sweeps vary one draw at a time. UK_SAMPLE_SEED_DEFAULT = 578 -( - _, - _CAPITAL_GAINS_CLONE_COLUMN, - _REGION_COLUMN, -) = SPI_REPLACEMENT_STRATA_COLUMNS - - -def _strict_bool(values: pd.Series, *, label: str) -> np.ndarray: - if values.isna().any(): - raise ValueError(f"UK sample: {label} contains missing values.") - if pd.api.types.is_bool_dtype(values): - return values.to_numpy(dtype=bool) - if pd.api.types.is_numeric_dtype(values): - numeric = pd.to_numeric(values, errors="coerce") - if not numeric.isin([0, 1]).all(): - raise ValueError( - f"UK sample: {label} must contain only boolean/0/1 values." - ) - return numeric.to_numpy(dtype=float) != 0.0 - raise ValueError(f"UK sample: {label} must be a boolean column.") - - -def _int_column(values: pd.Series, *, label: str) -> np.ndarray: - if values.isna().any(): - raise ValueError(f"UK sample: {label} contains missing values.") - if not pd.api.types.is_integer_dtype(values.dtype): - raise ValueError( - f"UK sample: {label} must be integer-typed; got dtype {values.dtype}." - ) - return values.to_numpy(dtype=np.int64) - - -def _str_column(values: pd.Series, *, label: str) -> np.ndarray: - if values.isna().any(): - raise ValueError(f"UK sample: {label} contains missing values.") - array = values.to_numpy(dtype=object) - if not all(isinstance(value, str) and value for value in array): - raise ValueError( - f"UK sample: {label} must contain non-empty strings; numeric " - "codes are refused because equal codes of different dtypes " - "(1 vs 1.0) would silently split one stratum into two." - ) - return array - - -_SPINE_SOURCE_HOUSEHOLD_ID_COLUMN = "source_household_id" -_SPINE_SUPPORT_CLONE_INDEX_COLUMN = "household_support_clone_index" -_SPINE_CGT_SUPPORT_COPY_COLUMN = "household_is_cgt_support_copy" -_SPINE_CGT_RESIDENTIAL_ARM_COLUMN = "household_is_cgt_residential_clone" - - -def uk_spine_source_family_units(frame: Frame) -> tuple[np.ndarray, np.ndarray]: - """Return the UK spine's explicit source-family keys and raw-row strata. - - Every derivative row travels with the raw FRS household named by its - ``source_household_id``. The raw household owns the family's region, so - derivative flags and incidental derivative regions never split one - source family across sampling strata. - """ - - household = frame.table("household") - household_id_column = "household_id" - lineage_columns = { - _SPINE_SOURCE_HOUSEHOLD_ID_COLUMN, - _SPINE_SUPPORT_CLONE_INDEX_COLUMN, - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - _CAPITAL_GAINS_CLONE_COLUMN, - _SPINE_CGT_SUPPORT_COPY_COLUMN, - _SPINE_CGT_RESIDENTIAL_ARM_COLUMN, - } - required = {household_id_column, _REGION_COLUMN, *lineage_columns} - missing = sorted(required - set(household.columns)) - present_lineage = lineage_columns & set(household.columns) - if not present_lineage: - raw_required = {household_id_column, _REGION_COLUMN} - raw_missing = sorted(raw_required - set(household.columns)) - if raw_missing: - raise ValueError( - "UK spine sample requires household identity and region columns; " - f"household is missing {raw_missing}." - ) - household_ids = _int_column( - household[household_id_column], label=household_id_column - ) - regions = _str_column(household[_REGION_COLUMN], label=_REGION_COLUMN) - return household_ids, np.asarray( - [f"region={region}" for region in regions], dtype=object - ) - if missing: - raise ValueError( - "UK spine sample has a partial lineage surface; " - f"household is missing {missing}." - ) - - household_ids = _int_column(household["household_id"], label="household_id") - units = _int_column( - household[_SPINE_SOURCE_HOUSEHOLD_ID_COLUMN], - label=_SPINE_SOURCE_HOUSEHOLD_ID_COLUMN, - ) - support_clone = _int_column( - household[_SPINE_SUPPORT_CLONE_INDEX_COLUMN], - label=_SPINE_SUPPORT_CLONE_INDEX_COLUMN, - ) - if (support_clone < 0).any(): - raise ValueError( - "UK sample: household_support_clone_index must be non-negative." - ) - spi = _strict_bool( - household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN], - label=HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - ) - capital_gains = _strict_bool( - household[_CAPITAL_GAINS_CLONE_COLUMN], - label=_CAPITAL_GAINS_CLONE_COLUMN, - ) - # A support split's root keeps its raw-row role; its created copies join - # the family through source_household_id (microcosm#1045). - support_copy = _strict_bool( - household[_SPINE_CGT_SUPPORT_COPY_COLUMN], - label=_SPINE_CGT_SUPPORT_COPY_COLUMN, - ) - # A residential split's incumbent arm keeps its raw-row role and ids; - # the residential arms join the family the same way (microcosm#1063). - residential_arm = _strict_bool( - household[_SPINE_CGT_RESIDENTIAL_ARM_COLUMN], - label=_SPINE_CGT_RESIDENTIAL_ARM_COLUMN, - ) - regions = _str_column(household[_REGION_COLUMN], label=_REGION_COLUMN) - - raw = ( - (support_clone == 0) & ~spi & ~capital_gains & ~support_copy & ~residential_arm - ) - source_rows = raw & (household_ids == units) - source_keys = set(int(value) for value in units[source_rows]) - missing_sources = len(set(int(value) for value in units) - source_keys) - if missing_sources: - raise ValueError( - f"UK spine sample: {missing_sources} source_household_id family " - "key(s) lack a household_id-bearing raw row." - ) - - raw_region_by_unit: dict[int, str] = {} - multi_region: list[int] = [] - for unit in sorted(set(int(value) for value in units)): - family_regions = set(str(value) for value in regions[raw & (units == unit)]) - if len(family_regions) > 1: - multi_region.append(unit) - else: - raw_region_by_unit[unit] = next(iter(family_regions)) - if multi_region: - raise ValueError( - f"UK spine sample: {len(multi_region)} source family(ies) span more " - "than one raw-row region." - ) - - strata = np.asarray( - [f"region={raw_region_by_unit[int(unit)]}" for unit in units], - dtype=object, - ) - return units, strata - - -def sample_uk_spine_frame( - frame: Frame, - *, - fraction: float, - seed: int, -) -> tuple[Frame, dict[str, object]]: - """Sample complete explicit source families from a UK Microcosm spine.""" - - validate_sample_fraction(fraction, label="UK spine sample") - validate_sample_seed(seed, label="UK spine sample") - validate_uk_national_frame(frame) - - household = frame.table("household") - units, strata = uk_spine_source_family_units(frame) - pre_counts = pd.Series(units).value_counts(sort=False) - full_mass = float(frame.weights_for("household").total) - sampled, core_receipt = sample_frame_households( - frame, - fraction=fraction, - seed=seed, - source_name="UK spine", - unit_ids=units, - unit_strata=strata, - unit_noun="source family", - floor_context="the UK rowwise candidate", - ) - normalized, factor = normalize_sampled_household_mass( - sampled, - target_mass=full_mass, - source_name="UK spine", - ) - - post_units, _ = uk_spine_source_family_units(normalized) - post_counts = pd.Series(post_units).value_counts(sort=False) - incomplete = sorted( - int(unit) - for unit, count in post_counts.items() - if int(count) != int(pre_counts.loc[unit]) - ) - if incomplete: - raise ValueError( - f"UK spine sample retained {len(incomplete)} incomplete source family(ies)." - ) - validate_uk_national_frame(normalized) - return normalized, { - "fraction": float(fraction), - "seed": int(seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS.get(float(fraction)), - "pre_household_count": int(len(household)), - "post_household_count": int(len(normalized.table("household"))), - "pre_family_count": int(len(pre_counts)), - "post_family_count": int(len(post_counts)), - "normalization_factor": float(factor), - "strata_count": int(len(np.unique(strata))), - "receipt": dict(core_receipt), - } - def sample_uk_spine_root( frame: Frame, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py index 39419f36a8..7f31002e30 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py @@ -347,13 +347,8 @@ def checkpoint_references(sidecar_path: Path | None) -> tuple[list[dict], str | ( "population", "Population pool", - "Checkpoint admission, sampling, normalisation, expansion into K " - "geographic copies and the pooled problem checkpoint.", - ), - ( - "geography", - "Geography", - "Location draws, derived geographies and the geography integrity gate.", + "Checkpoint admission and the pool checkpoint: the bound spine whole, " + "with its geographic clones, areas and sampling.", ), ( "targets", @@ -381,21 +376,8 @@ def checkpoint_references(sidecar_path: Path | None) -> tuple[list[dict], str | _FULL_BUILD_GROUPS = { "population": ( "uk.full.spine_checkpoint", - "uk.full.sample", - "uk.full.normalize", - "uk.full.expand", "uk.full.pool", ), - "geography": ( - "uk.full.identity", - "uk.full.locations", - "uk.full.geography_mapping", - "uk.full.geography_gate", - "uk.full.geography.assign", - "uk.full.geography.derive", - "uk.full.geography.local_authority", - "uk.full.geography.gate", - ), "targets": ( "uk.full.target_compilation", "uk.full.target_selection", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py index 30e781c75c..1fd2a05afb 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py @@ -107,6 +107,7 @@ "record_candidate_error", "refuse_dense_role_arguments", "refuse_national_role_arguments", + "refuse_spine_bound_arguments", "release_verdict", "resolve_role_arguments", "size_doctrine_drift", @@ -265,7 +266,12 @@ def _size_l2_parameters(args: argparse.Namespace) -> dict[str, Any]: def rowwise_parameters(args: argparse.Namespace, *, source_year: int) -> dict[str, Any]: - """The run parameters a rowwise manifest, plan and identity digest record.""" + """The run parameters a rowwise manifest, plan and identity digest record. + + On the dense line ``n_clones``, ``sample_fraction`` and ``sample_seed`` + are the bound spine's: the build's preparation sets them from the spine + before anything records them (microcosm#931). + """ posture = posture_of(args) return { @@ -295,7 +301,6 @@ def rowwise_parameters(args: argparse.Namespace, *, source_year: int) -> dict[st if args.resume_size_checkpoint is None else str(args.resume_size_checkpoint.expanduser().resolve()), "source_year": source_year, - "source_lineage_modulus": args.source_lineage_modulus, "sample_fraction": float(args.sample_fraction), "sample_seed": int(args.sample_seed), "engine_blocks": int(args.engine_blocks), @@ -429,31 +434,6 @@ def record_candidate_error( #: Solve arguments whose argparse default is ``None`` so an explicit value can #: be told from the role's default: the other role's refusal table keys on #: what was actually given. -#: CLI labels of the three UK atomic-area supports (microcosm#932), in -#: ``atomic_area_support.SYSTEMS`` order; ``full_build_cli`` declares the -#: ``--atomic-support-